src/aptRepo.ts

@@ -198,8 +198,8 @@ async function mainConfig(configPath: string) {
   const config = await getConfig(configPath);
   const packageReg = new localRegistryManeger();
   Promise.all(config.repos.map(async repo => {
-    if (repo.from === "release") return release.fullConfig({config: repo.repo, githubToken: repo?.auth?.password}, packageReg).catch(console.error);
-    if (repo.from === "oci") return ghcr.fullConfig({image: repo.repo, targetInfo: repo.ociConfig}, packageReg).catch(console.error);
+    if (repo.from === "release") return release.fullConfig({config: repo.repo, githubToken: repo?.auth?.password}, () => {}).catch(console.error);
+    if (repo.from === "oci") return ghcr.fullConfig({image: repo.repo, targetInfo: repo.ociConfig}, () => {}).catch(console.error);
   }));
   return packageReg;
 }

src/apt_repo_v2.ts

@@ -0,0 +1,214 @@
+import { Readable } from "node:stream";
+import { extendsCrypto } from "@sirherobrine23/coreutils";
+import { format } from "node:util";
+import * as lzma from "lzma-native";
+import express from "express";
+import zlib from "node:zlib";
+import { packageControl } from "./deb.js";
+
+type registerOobject = {
+  [packageName: string]: {
+    getStream: () => Promise<Readable>,
+    control: packageControl,
+    from?: string,
+  }[]
+};
+
+export function packageManeger(RootOptions?: {origin?: string, lebel?: string}) {
+  const localRegister: registerOobject = {};
+  function pushPackage(control: packageControl, getStream: () => Promise<Readable>, from?: string) {
+    if (!localRegister[control.Package]) localRegister[control.Package] = [];
+    localRegister[control.Package].push({
+      getStream,
+      control,
+      from,
+    });
+  }
+
+  function getPackages() {
+    return localRegister;
+  }
+
+  async function createPackages(options?: {packageName?: string, Arch?: string}, streams?: (data: {gz: zlib.Gzip, xz: lzma.JSLzmaStream, raw: Readable}) => void) {
+    if (options?.packageName === "all") options.packageName = undefined;
+    const sizes = {gz: 0, xz: 0, raw: 0};
+    const raw = new Readable();
+    const rawHASH = extendsCrypto.createSHA256_MD5(raw, "both", new Promise(resolve => raw.on("end", resolve)));
+    raw.on("data", chunck => sizes.raw += chunck.length);
+    const gz = raw.pipe(zlib.createGzip());
+    const gzHASH = extendsCrypto.createSHA256_MD5(gz, "both", new Promise(resolve => gz.on("end", resolve)));
+    gz.on("data", chunck => sizes.gz += chunck.length);
+    const xz = raw.pipe(lzma.createCompressor());
+    const xzHASH = extendsCrypto.createSHA256_MD5(xz, "both", new Promise(resolve => xz.on("end", resolve)));
+    xz.on("data", chunck => sizes.xz += chunck.length);
+    if (streams) streams({gz, xz, raw});
+
+    const writeObject = (packageData: (typeof localRegister)[string][number]) => {
+      const control = packageData.control;
+      control.Filename = format("pool/%s/%s/%s.deb", control.Package, control.Architecture, control.Version);
+      const desc = control.Description;
+      delete control.Description;
+      control.Description = desc;
+      const data = Buffer.from(Object.keys(control).map(key => `${key}: ${control[key]}`).join("\n") + "\n\n", "utf8");
+      raw.push(data);
+    }
+
+    if (!!options?.packageName) {
+      const packageVersions = localRegister[options?.packageName];
+      if (!packageVersions) {
+        raw.push(null);
+        raw.destroy();
+        throw new Error("Package not found");
+      }
+      for (const packageData of packageVersions) {
+        if (options?.Arch && packageData.control.Architecture !== options?.Arch) continue;
+        writeObject(packageData);
+      }
+    } else {
+      for (const packageName in localRegister) {
+        for (const packageData of localRegister[packageName]) {
+          if (options?.Arch && packageData.control.Architecture !== options?.Arch) continue;
+          writeObject(packageData);
+        }
+      }
+    }
+
+    raw.push(null);
+    // raw.end();
+    // raw.destroy();
+
+    return {
+      raw: {
+        ...(await rawHASH),
+        size: sizes.raw,
+      },
+      gz: {
+        ...(await gzHASH),
+        size: sizes.gz,
+      },
+      xz: {
+        ...(await xzHASH),
+        size: sizes.xz,
+      }
+    };
+  }
+
+  async function createRelease(options?: {packageName?: string, Arch?: string, includesHashs?: boolean}) {
+
+    const textLines = [
+      `Lebel: ${RootOptions?.lebel||"node-apt"}`,
+      `Date: ${new Date().toUTCString()}`
+    ];
+
+    if (options?.packageName) {
+      const packageData = localRegister[options?.packageName];
+      if (!packageData) throw new Error("Package not found");
+      const archs = [...(new Set(localRegister[options?.packageName].map((p) => p.control.Architecture)))];
+      const components = [...(new Set(localRegister[options?.packageName].map((p) => p.control.Section||"main")))];
+      textLines.push(`Suite: ${options?.packageName}`);
+      textLines.push(`Architectures: ${archs.filter(arch => !options?.Arch ? true : arch === options?.Arch).join(" ")}`);
+      textLines.push(`Components: ${components.join(" ")}`);
+      if (options?.includesHashs) {
+        const Hashs = await createPackages({packageName: options?.packageName});
+        textLines.push(`MD5Sum:`);
+        textLines.push(`  ${Hashs.raw.md5}  ${Hashs.raw.size}  main/binary-${options?.Arch||"all"}/Packages`);
+        textLines.push(`  ${Hashs.xz.md5}  ${Hashs.xz.size}  main/binary-${options?.Arch||"all"}/Packages.xz`);
+        textLines.push(`  ${Hashs.gz.md5}  ${Hashs.gz.size}  main/binary-${options?.Arch||"all"}/Packages.gz`);
+        textLines.push(`SHA256:`);
+        textLines.push(`  ${Hashs.raw.sha256}  ${Hashs.raw.size}  main/binary-${options?.Arch||"all"}/Packages`);
+        textLines.push(`  ${Hashs.xz.sha256}  ${Hashs.xz.size}  main/binary-${options?.Arch||"all"}/Packages.xz`);
+        textLines.push(`  ${Hashs.gz.sha256}  ${Hashs.gz.size}  main/binary-${options?.Arch||"all"}/Packages.gz`);
+      }
+    } else {
+      // For all packages
+      // const archs = [...(new Set(Object.values(localRegister).flat().map((p) => p.control.Architecture)))];
+      // textLines.push(`Suite: ${options?.packageName}`);
+      // textLines.push(`Architectures: ${archs.filter(arch => !options?.Arch ? true : arch === options?.Arch).join(" ")}`);
+      // textLines.push("Components: main");
+      // if (options?.includesHashs) {
+      //   const Hashs = await createPackages();
+      //   textLines.push(`MD5Sum:`);
+      //   textLines.push(`  ${Hashs.raw.md5}  ${Hashs.raw.size}  main/binary-${options?.Arch||"all"}/Packages`);
+      //   textLines.push(`  ${Hashs.xz.md5}  ${Hashs.xz.size}  main/binary-${options?.Arch||"all"}/Packages.xz`);
+      //   textLines.push(`  ${Hashs.gz.md5}  ${Hashs.gz.size}  main/binary-${options?.Arch||"all"}/Packages.gz`);
+      //   textLines.push(`SHA256:`);
+      //   textLines.push(`  ${Hashs.raw.sha256}  ${Hashs.raw.size}  main/binary-${options?.Arch||"all"}/Packages`);
+      //   textLines.push(`  ${Hashs.xz.sha256}  ${Hashs.xz.size}  main/binary-${options?.Arch||"all"}/Packages.xz`);
+      //   textLines.push(`  ${Hashs.gz.sha256}  ${Hashs.gz.size}  main/binary-${options?.Arch||"all"}/Packages.gz`);
+      // }
+      throw new Error("Not implemented");
+    }
+    textLines.push("\n");
+    // convert to string
+    return textLines.join("\n");
+  }
+
+  return {
+    getPackages,
+    pushPackage,
+    createRelease,
+    createPackages,
+  };
+}
+
+export default async function repo(aptConfig: {}) {
+  const app = express();
+  const registry = packageManeger();
+  app.disable("x-powered-by").disable("etag").use(express.json()).use(express.urlencoded({extended: true})).use((_req, res, next) => {
+    res.json = (data) => res.setHeader("Content-Type", "application/json").send(JSON.stringify(data, null, 2));
+    next();
+  }).use((req, _res, next) => {
+    next();
+    return console.log(`${new Date().toISOString()} ${req.method} ${req.url}`);
+  });
+
+  app.get("/", (_req, res) => res.json(registry.getPackages()));
+
+  app.get("/sources.list", (req, res) => {
+    res.setHeader("Content-type", "text/plain");
+    let config = "";
+    if (req.query.all) config += format("deb [trusted=yes] %s://%s %s main\n", req.protocol, req.headers.host, "all");
+    else {
+      for (const suite of Object.keys(registry.getPackages())) {
+        config += format("deb [trusted=yes] %s://%s %s main\n", req.protocol, req.headers.host, suite);
+      }
+    }
+    res.send(config+"\n");
+  });
+
+  // apt /dists
+  // release
+  app.get("/dists/:suite/InRelease", (_req, res) => res.status(400).json({error: "Not implemented, required pgp to auth"}));
+  app.get("/dists/:suite/Release", (req, res, next) => {
+    const { suite } = req.params;
+    res.setHeader("Content-Type", "text/plain");
+    return registry.createRelease({packageName: suite === "all"?undefined:suite, includesHashs: true}).then((release) => res.send(release)).catch(next);
+  });
+
+  // Components
+  app.get("/dists/:suite/:component/binary-:arch/Packages(.(gz|xz)|)", (req, res, next) => {
+    const {suite, arch} = req.params;
+    registry.createPackages({packageName: suite, Arch: arch}, (streamers) => {
+      if (req.path.endsWith(".gz")) {
+        streamers.gz.pipe(res.writeHead(200, {"Content-Encoding": "application/x-gzip"}));
+      } else if (req.path.endsWith(".xz")) {
+        streamers.xz.pipe(res.writeHead(200, {"Content-Encoding": "application/x-xz"}));
+      } else {
+        streamers.raw.pipe(res.writeHead(200, {"Content-Encoding": "text/plain"}));
+      }
+    }).catch(next);
+  });
+
+  // No Page
+  app.use((err, _req, res, _next) => {
+    console.log("Error: %s, req path: %s", err?.message||err, _req.path);
+    return res.status(500).json({
+      error: err?.message||err
+    });
+  });
+
+  return {
+    app,
+    registry,
+  };
+}

src/deb.ts

@@ -0,0 +1,71 @@
+import tar from "tar";
+import { createExtract } from "./ar.js";
+import { Decompressor } from "lzma-native";
+import { Readable } from "stream";
+import { extendsCrypto } from "@sirherobrine23/coreutils";
+
+export type packageControl = {
+  Package: string
+  Version: string,
+  /** endpoint folder file */
+  Filename: string,
+  "Installed-Size": number,
+  Maintainer: string,
+  Architecture: string,
+  Depends?: string,
+  Homepage?: string,
+  Section?: string,
+  Priority?: string,
+  Size: number,
+  MD5sum: string,
+  SHA256: string,
+  Description?: string,
+};
+
+export function parseControl(rawControlFile: string) {
+  const controlObject = {};
+  for (const line of rawControlFile.split(/\r?\n/)) {
+    if (/^[\w\S]+:/.test(line)) {
+      const [, key, value] = line.match(/^([\w\S]+):(.*)$/);
+      controlObject[key.trim()] = value.trim();
+    } else {
+      const latestKey = Object.keys(controlObject).at(-1);
+      controlObject[latestKey] += "\n";
+      controlObject[latestKey] += line;
+    }
+  }
+  return controlObject as packageControl;
+}
+
+export type debReturn = Awaited<ReturnType<typeof extractDebControl>>;
+
+export async function extractDebControl(debStream: Readable) {
+  return new Promise<{size: number, control: packageControl}>((done, reject) => {
+    let fileSize = 0;
+    debStream.on("data", (chunk) => fileSize += chunk.length);
+    const signs = extendsCrypto.createSHA256_MD5(debStream, "both", new Promise(done => debStream.once("end", done)));
+    return debStream.pipe(createExtract((info, stream) => {
+      if (!(info.name.endsWith("control.tar.gz")||info.name.endsWith("control.tar.xz"))) return;
+      (info.name.endsWith("tar.gz")?stream:stream.pipe(Decompressor())).pipe(tar.list({
+        onentry(controlEntry) {
+          if (!controlEntry.path.endsWith("control")) return null;
+          let controlFile: Buffer;
+          controlEntry.on("data", chunck => controlFile = (!controlFile)?chunck:Buffer.concat([controlFile, chunck])).once("end", async () => {
+            const sign = await signs;
+            const control = parseControl(controlFile.toString());
+            debStream.on("end", () => {
+              control.MD5sum = sign.md5;
+              control.SHA256 = sign.sha256;
+              control.Size = fileSize;
+              return done({
+                control,
+                size: fileSize
+              });
+            });
+          }).on("error", reject);
+        },
+      // @ts-ignore
+      })).on("error", reject);
+    })).on("error", reject);
+  });
+}

src/githubRelease.ts

@@ -1,69 +1,31 @@
-import { parseDebControl } from "./aptRepo.js";
-import { createExtract } from "./ar.js";
-import coreUtils, { extendsCrypto } from "@sirherobrine23/coreutils";
-import tar from "tar";
-import { localRegistryManeger } from "./aptRepo.js";
-import { format } from "util";
-import { Decompressor } from "lzma-native";
+import coreUtils from "@sirherobrine23/coreutils";
+import { Readable } from "stream";
+import { extractDebControl, debReturn } from "./deb.js";
+export type baseOptions<T extends {} = {}> = {repo: string, owner: string} & T;
 
-export type baseOptions<T extends {} = {}> = {
-  repo: string,
-  owner: string
-} & T;
-
-export async function list(config: string|baseOptions<{releaseTag?: string}>, githubToken?: string) {
-  if (typeof config === "string") {
-    const [owner, repo] = config.split("/");
-    config = {
-      owner,
-      repo
-    };
+export default fullConfig;
+export async function fullConfig(config: {config: string|baseOptions<{releaseTag?: string}>, githubToken?: string}, fn: (data: debReturn & {getStream: () => Promise<Readable>}) => void) {
+  if (typeof config.config === "string") {
+    const [owner, repo] = config.config.split("/");
+    config.config = {owner, repo};
   }
 
-  const options: baseOptions<{releaseTag?: string}> = config;
+  const options: baseOptions<{releaseTag?: string}> = config.config;
   const releases = (await coreUtils.httpRequestGithub.GithubRelease(options.owner, options.repo)).slice(0, 10).filter(data => data.assets.some(file => file.name.endsWith(".deb"))).map(data => {
     return {
       tag: data.tag_name,
       assets: data.assets.filter(data => data.name.endsWith(".deb")).map(({name, browser_download_url}) => ({name, download: browser_download_url}))
     };
-  });
+  }).filter(({assets}) => assets?.length > 0);
 
-  return releases.filter(({assets}) => assets?.length > 0);
-}
-
-export async function fullConfig(config: {config: string|baseOptions<{releaseTag?: string}>, githubToken?: string}, packageManeger: localRegistryManeger) {
-  const releases = await list(config.config, config.githubToken);
-  for (const {assets, tag} of releases ?? []) for (const {download} of assets ?? []) {
-    let size = 0;
-      const request = (await coreUtils.httpRequest.pipeFetch(download)).on("data", (chunk) => size += chunk.length);
-      const signs = extendsCrypto.createSHA256_MD5(request, "both", new Promise(done => request.on("end", done)));
-      request.pipe(createExtract((info, stream) => {
-        if (!(info.name.endsWith("control.tar.gz")||info.name.endsWith("control.tar.xz"))) return;
-        (info.name.endsWith("tar.gz")?stream:stream.pipe(Decompressor())).pipe(tar.list({
-          onentry: (tarEntry) => {
-            if (!tarEntry.path.endsWith("control")) return;
-            let controlBuffer: Buffer;
-            tarEntry.on("data", (chunk) => {
-              if (!controlBuffer) controlBuffer = chunk;
-              else controlBuffer = Buffer.concat([controlBuffer, chunk]);
-            }).on("error", console.log);
-            request.on("end", async () => {
-              const debConfig = parseDebControl(controlBuffer);
-              if (!(debConfig.Package && debConfig.Version && debConfig.Architecture)) return;
-              const sigs = await signs;
-              packageManeger.registerPackage({
-                name: debConfig.Package,
-                version: debConfig.Version,
-                arch: debConfig.Architecture,
-                packageConfig: debConfig,
-                signature: sigs,
-                size,
-                from: format("github release, tag: %s, repo: %s", tag, config.config),
-                getStrem: async () => coreUtils.httpRequest.pipeFetch(download),
-              });
-            }).on("error", console.log);
-          }
-        }));
-      })).on("error", console.log);
+  for (const rel of releases) {
+    for (const asset of rel.assets) {
+      const getStream = async () => coreUtils.httpRequest.pipeFetch(asset.download)
+      const control = await extractDebControl(await getStream());
+      fn({
+        ...control,
+        getStream,
+      });
+    }
   }
 }

src/index.ts

@@ -1,15 +1,23 @@
 #!/usr/bin/env node
 import yargs from "yargs";
-import { createAPI } from "./aptRepo.js";
+import repo from "./apt_repo_v2.js";
+import { getConfig } from "./repoConfig.js";
+import github_release from "./githubRelease.js";
+import oci_registry from "./oci_registry.js";
+
 yargs(process.argv.slice(2)).wrap(null).strict().help().option("cofig-path", {
   type: "string",
   default: process.cwd()+"/repoconfig.yml",
 }).option("port", {
   type: "number",
   default: 3000,
-}).parseAsync().then(options => {
-  return createAPI({
-    configPath: options["cofig-path"],
-    portListen: options.port,
+}).parseAsync().then(async options => {
+  const { app, registry } = await repo({});
+  app.listen(options.port, () => {
+    console.log(`Server listening on port ${options.port}`);
   });
+  Promise.all((await getConfig(options["cofig-path"])).repos.map(async repo => {
+    if (repo.from === "oci") return oci_registry({image: repo.repo, targetInfo: repo.ociConfig}, data => registry.pushPackage(data.control, data.getStream)).catch(console.error);
+    else if (repo.from === "release") return github_release({config: repo.repo, githubToken: repo.auth.password}, data => registry.pushPackage(data.control, data.getStream)).catch(console.error);
+  })).catch(console.error);
 });

src/oci_registry.ts

@@ -1,12 +1,10 @@
-import { localRegistryManeger, parseDebControl } from "./aptRepo.js";
-import { DockerRegistry, extendsCrypto } from "@sirherobrine23/coreutils";
-import { createExtract } from "./ar.js";
+import { debReturn, extractDebControl } from "./deb.js";
+import { DockerRegistry } from "@sirherobrine23/coreutils";
 import { Readable } from "stream";
 import tar from "tar";
-import { Decompressor } from "lzma-native";
-import { format } from "util";
 
-export async function fullConfig(imageInfo: {image: string, targetInfo?: DockerRegistry.Manifest.platfomTarget}, packageManeger: localRegistryManeger) {
+export default fullConfig;
+export async function fullConfig(imageInfo: {image: string, targetInfo?: DockerRegistry.Manifest.platfomTarget}, fn: (data: debReturn & {getStream: () => Promise<Readable>}) => void) {
   const registry = await DockerRegistry.Manifest.Manifest(imageInfo.image, imageInfo.targetInfo);
   await registry.layersStream((data) => {
     if (!(["gzip", "gz", "tar"]).some(ends => data.layer.mediaType.endsWith(ends))) {
@@ -14,44 +12,19 @@ export async function fullConfig(imageInfo: {image: string, targetInfo?: DockerR
       return null;
     }
     return data.stream.pipe(tar.list({
-      onentry(entry) {
+      async onentry(entry) {
         if (!entry.path.endsWith(".deb")) return null;
-        let fileSize = 0;
-        entry.on("data", (chunk) => fileSize += chunk.length);
-        const signs = extendsCrypto.createSHA256_MD5(entry as any, "both", new Promise(done => entry.once("end", done)));
-        return entry.pipe(createExtract((info, stream) => {
-          if (!(info.name.endsWith("control.tar.gz")||info.name.endsWith("control.tar.xz"))) return;
-          (info.name.endsWith("tar.gz")?stream:stream.pipe(Decompressor())).pipe(tar.list({
-            onentry(controlEntry) {
-              if (!controlEntry.path.endsWith("control")) return null;
-              let controlFile: Buffer;
-              controlEntry.on("data", chunck => controlFile = (!controlFile)?chunck:Buffer.concat([controlFile, chunck])).once("end", async () => {
-                const sign = await signs;
-                const control = parseDebControl(controlFile);
-                entry.on("end", () => {
-                  packageManeger.registerPackage({
-                    name: control.Package,
-                    version: control.Version,
-                    arch: control.Architecture,
-                    packageConfig: control,
-                    size: fileSize,
-                    signature: sign,
-                    from: format("oci registry, image: %s, layer: %s", imageInfo.image, data.layer.digest),
-                    getStrem: () => new Promise<Readable>(done => {
-                      registry.blobLayerStream(data.layer.digest).then((stream) => stream.pipe(tar.list({
-                        onentry(getEntry) {
-                          if (getEntry.path !== entry.path) return;
-                          done(getEntry as any);
-                        }
-                      })));
-                    }),
-                  });
-                });
-              }).on("error", console.error);
-            },
-          // @ts-ignore
-          })).on("error", console.error);
-        })).on("error", console.log);
+        const control = await extractDebControl(entry as any);
+        return fn({
+          ...control,
+          getStream: async () => {
+            return new Promise<Readable>((done, reject) => registry.blobLayerStream(data.layer.digest).then(stream => stream.pipe(tar.list({
+              onentry(getEntry) {
+                if (getEntry.path === entry.path) return done(getEntry as any);
+              }
+            }))).catch(reject));
+          },
+        });
       },
     }));
   });