.github/workflows/codeql.yml

@@ -1,72 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ "main" ]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines.
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
-      with:
-        category: "/language:${{matrix.language}}"

.vscode/launch.json

@@ -11,8 +11,12 @@
       "internalConsoleOptions": "openOnSessionStart",
       "skipFiles": ["<node_internals>/**", "node_modules/**"],
       "cwd": "${workspaceRoot}",
-      "runtimeExecutable": "ts-node",
+      "runtimeExecutable": "node",
-      "args": ["src/index.ts", "server"]
+      "args": ["src/index.js", "server", "--cpus=0"],
+      "preLaunchTask": {
+        "type": "npm",
+        "script": "build"
+      }
     }
   ]
 }

Dockerfile

@@ -1,8 +1,7 @@
-FROM node:latest
+FROM node:lts-alpine
 WORKDIR /app
 COPY package*.json ./
 RUN npm ci
 COPY . .
-RUN npm run build
+RUN npm run build && npm link
-ENV NODE_OPTIONS="--max_old_space_size=4096"
+ENTRYPOINT [ "apt-stream", "server" ]
-ENTRYPOINT [ "node", "src/index.js", "server" ]

package.json

@@ -30,14 +30,15 @@
     "apt-stream": "./src/index.js"
   },
   "scripts": {
-    "start": "ts-node src/index.ts",
+    "start": "npm run build && node ./src/index.js",
-    "build": "tsc"
+    "dev": "npm run build && node --inspect ./src/index.js server --cpu=0",
+    "build": "tsc --build --clean && tsc"
   },
   "devDependencies": {
-    "@types/cron": "^2.0.0",
     "@types/express": "^4.17.15",
     "@types/lzma-native": "^4.0.1",
     "@types/node": "^18.11.18",
+    "@types/semver": "^7.3.13",
     "@types/tar": "^6.1.3",
     "@types/yargs": "^17.0.19",
     "ts-node": "^10.9.1",
@@ -45,10 +46,11 @@
   },
   "dependencies": {
     "@sirherobrine23/coreutils": "^2.2.5",
-    "cron": "^2.1.0",
     "express": "^4.18.2",
     "lzma-native": "^8.0.6",
+    "mongodb": "^4.13.0",
     "openpgp": "^5.5.0",
+    "semver": "^7.3.8",
     "tar": "^6.1.13",
     "yaml": "^2.2.1",
     "yargs": "^17.6.2"

src/express_route.ts

@@ -1,211 +1,236 @@
-import coreUtils, { DebianPackage, httpRequestGithub, httpRequest, DockerRegistry, extendFs } from "@sirherobrine23/coreutils";
+import { Compressor as lzmaCompress } from "lzma-native";
-import { getConfig, distManegerPackages } from "./repoConfig.js";
+import { createGzip } from "node:zlib";
-import { createReadStream, createWriteStream, watchFile, promises as fs } from "node:fs";
+import { getConfig } from "./repoConfig.js";
-import { getPackages } from "./mirror.js";
 import { Readable } from "node:stream";
-import { CronJob } from "cron";
+import package_maneger, { packageSave } from "./packagesData.js";
-import { format } from "node:util";
+import coreUtils from "@sirherobrine23/coreutils";
+import cluster from "node:cluster";
 import express from "express";
 import openpgp from "openpgp";
-import tar from "tar";
+import semver from "semver";
 import path from "node:path";
+import os from "node:os";
 
-export default async function main(configPath: string) {
+export default async function initApp(config: string) {
-  // Load config
+  const packageConfig = await getConfig(config);
-  const packInfos = new distManegerPackages()
+  const packageManeger = await package_maneger(packageConfig);
-  let repositoryConfig = await getConfig(configPath);
-
-  // Express app
   const app = express();
-  app.disable("x-powered-by").disable("etag").use(express.json()).use(express.urlencoded({ extended: true })).use((req, res, next) => {
+  app.disable("x-powered-by").disable("etag").use(express.json()).use(express.urlencoded({extended: true})).use((req, res, next) => {
-    res.json = (data) => res.setHeader("Content-Type", "application/json").send(JSON.stringify(data, null, 2));
+    res.json = data => {
-    const requestInitial = Date.now();
+      Promise.resolve(data).then(data => res.setHeader("Content-Type", "application/json").send(JSON.stringify(data, null, 2))).catch(next);
-    console.log("[%s]: Method: %s, From: %s, Path %s", requestInitial, req.method, req.ip, req.path);
+      return res;
-    res.once("close", () => console.log("[%s]: Method: %s, From: %s, Path %s, Status: %s, Time: %sms", Date.now(), req.method, req.ip, req.path, res.statusCode, Date.now() - requestInitial));
+    };
+    const cluserID = (cluster.worker?.id === 1 ? "Primary" : cluster.worker?.id) ?? "Primary";
+    console.log("[%s, cluserID: %s]: Path: %s, Method: %s, IP: %s", new Date().toISOString(), cluserID, req.path, req.method, req.ip);
+    res.on("close", () => console.log("[%s, cluserID: %s]: Path: %s, Method: %s, IP: %s, Status: %f", new Date().toISOString(), cluserID, req.path, req.method, req.ip, res.statusCode));
     next();
   });
 
-  // Public key
+  // Info
-  app.get(["/public_key", "/public.gpg"], async ({res}) => {
+  app.get("/", ({res}) => {
-    const Key = repositoryConfig["apt-config"]?.pgpKey;
+    return res.json({
-    if (!Key) return res.status(400).json({error: "This repository no sign Packages files"});
+      cluster: cluster.worker?.id ?? "No clustered",
-    const pubKey = (await openpgp.readKey({ armoredKey: Key.public })).armor();
+      cpuCores: os.cpus().length || "Unknown CPU core",
-    return res.setHeader("Content-Type", "application/pgp-keys").send(pubKey);
+      hostArch: process.arch,
+      hostPlatform: process.platform,
+      nodeVersion: process.version,
+    });
   });
 
-  // Sources list
+  app.get("/pool/:dist/:suite/:package/:arch/:version/download.deb", async ({params: {dist, suite, package: packageName, arch, version}}, res, next) => {
-  app.get(["/source_list", "/sources.list"], (req, res) => {
+    try {
-    const remotePath = path.posix.resolve(req.baseUrl + req.path, ".."),
+      const data = (await packageManeger.getPackages(dist, suite, packageName, arch, version))?.at(-1);
-      protocol = req.headers["x-forwarded-proto"] ?? req.protocol,
+      if (!data) return next(new Error("Not Found"));
-      hostname = process.env["RAILWAY_STATIC_URL"] ?? `${req.hostname}:${req.socket.localPort}`,
+      const fileStream = await data.getFileStream();
-      host = repositoryConfig["apt-config"]?.sourcesHost ?? `${protocol}://${hostname}${remotePath}`,
+      fileStream.pipe(res.writeHead(200, {
-      concatPackage = packInfos.getAllDistribuitions(),
+        "Content-Type": "application/x-debian-package",
-      type = req.query.type ?? req.query.t,
+        "Content-Length": data.control.Size,
-      Conflicting = !!(req.query.conflicting ?? req.query.c);
+        "Content-Disposition": `attachment; filename="${packageName}_${version}_${arch}.deb"`,
-    if (type === "json") {
+        "SHA256_hash": data.control.SHA256,
-      return res.json({
+        "MD5Sum_hash": data.control.MD5sum
-        host,
+      }));
-        distribuitions: concatPackage
+    } catch (err) {
-      });
+      next(err);
-    } else if (type === "deb822") {}
-    let sourcesList = "";
-    concatPackage.forEach((dist) => sourcesList += format("deb %s %s %s\n", host, (Conflicting ? "./" : "")+dist.dist, dist.suites.join(" ")));
-    return res.status(200).setHeader("Content-Type", "text/plain").send(sourcesList);
-  });
-
-  // Download
-  app.get(["/pool", "/"], (_req, res) => res.json(packInfos.getAllDistribuitions()));
-  app.get("/pool/:dist", (req, res) => res.json(packInfos.getDistribuition(req.params.dist)));
-  app.get("/pool/:dist/:suite", ({params: {dist, suite}}, res) => res.json(packInfos.getPackageInfo({dist, suite})));
-  app.get("/pool/:dist/:suite/:arch", ({params: {dist, suite, arch}}, res) => res.json(packInfos.getPackageInfo({dist, suite, arch})));
-  app.get("/pool/:dist/:suite/:arch/:packageName", ({params: {dist, suite, arch, packageName}}, res) => res.json(packInfos.getPackageInfo({dist, suite, arch, packageName})));
-  app.get("/pool/:dist/:suite/:arch/:packageName/:version", ({params: {dist, suite, arch, packageName, version}}, res) => res.json(packInfos.getPackageInfo({dist, suite, arch, packageName, version})));
-  app.get("/pool/:dist/:suite/:arch/:packageName/:version/download.deb", async ({params: {dist, suite, arch, packageName, version}}, res, next) => packInfos.getPackageStream(dist, suite, arch, packageName, version).then(data => data.stream.pipe(res.writeHead(200, {"Content-Type": "application/x-debian-package", "Content-Length": data.control.Size, "Content-Disposition": `attachment; filename="${packageName}_${version}_${arch}.deb"`, "SHA256_hash": data.control.SHA256, "MD5Sum_hash": data.control.MD5sum}))).catch(next));
-
-  app.get("/dists/(./)?:dist/:suite/binary-:arch/Packages(.(xz|gz)|)", (req, res) => {
-    if (req.path.endsWith(".gz")) {
-      packInfos.createPackages({
-        compress: "gzip",
-        dist: req.params.dist,
-        arch: req.params.arch,
-        suite: req.params.suite,
-        writeStream: res.writeHead(200, {
-          "Content-Encoding": "gzip",
-          "Content-Type": "application/x-gzip"
-        }),
-      });
-    } else if (req.path.endsWith(".xz")) {
-      packInfos.createPackages({
-        compress: "xz",
-        dist: req.params.dist,
-        arch: req.params.arch,
-        suite: req.params.suite,
-        writeStream: res.writeHead(200, {
-          "Content-Encoding": "xz",
-          "Content-Type": "application/x-xz"
-        }),
-      });
-    } else {
-      packInfos.createPackages({
-        dist: req.params.dist,
-        arch: req.params.arch,
-        suite: req.params.suite,
-        writeStream: res.writeHead(200, {
-          "Content-Type": "text/plain"
-        }),
-      });
     }
   });
+  app.get("/pool/:dist/:suite/:package/:arch/:version", (req, res, next) => packageManeger.getPackages(req.params.dist, req.params.suite, req.params.package, req.params.arch, req.params.version).then(data => res.json(data.at(-1).control)).catch(next));
+  app.get("/pool/:dist/:suite/:package/:arch", (req, res, next) => packageManeger.getPackages(req.params.dist, req.params.suite, req.params.package, req.params.arch).then(data => res.json(data.map(({control}) => control))).catch(next));
+  app.get("/pool/:dist/:suite/:package", (req, res, next) => packageManeger.getPackages(req.params.dist, req.params.suite, req.params.package).then(data => res.json(data.map(({control}) => control))).catch(next));
+  app.get("/pool/:dist/:suite", (req, res, next) => packageManeger.getPackages(req.params.dist, req.params.suite).then(data => res.json(data.map(x => x.control))).catch(next));
+  app.get("/pool/:dist", (req, res, next) => packageManeger.getPackages(req.params.dist).then(data => res.json(data.reduce((old, current) => {
+    if (!old[current.suite]) old[current.suite] = [];
+    old[current.suite].push(current.control);
+    return old;
+  }, {}))).catch(next));
 
+  app.get("/pool", ({}, res, next) => packageManeger.getPackages().then(data => res.json(data.reduce((old, current) => {
+    if (!old[current.dist]) old[current.dist] = {};
+    if (!old[current.dist][current.suite]) old[current.dist][current.suite] = [];
+    old[current.dist][current.suite].push(current.control);
+    return old;
+  }, {}))).catch(next));
+
+  // Package list
+  app.get("/source(s)?(.list)?", async (req, res, next) => {
+    try {
+      const dists = await Promise.all(await packageManeger.getDists().then(data => data.map(dist => packageManeger.getDistInfo(dist).then(data => ({dist, data})))));
+      res.set("Content-Type", "text/plain");
+      const remotePath = path.posix.resolve(req.baseUrl + req.path, ".."), protocol = req.headers["x-forwarded-proto"] ?? req.protocol, hostname = process.env["RAILWAY_STATIC_URL"] ?? `${req.hostname}:${req.socket.localPort}`, host = packageConfig["apt-config"]?.sourcesHost ?? `${protocol}://${hostname}${remotePath}`;
+      const data = [];
+      for (const {dist, data: {suites}} of dists) data.push(`deb ${host} ${dist} ${suites.join(" ")}`);
+      return res.send(data.join("\n"));
+    } catch (err) {return next(err);}
+  });
+
+  // Dists info
+  app.get("/dists", ({}, res, next) => packageManeger.getDists().then(data => res.json(data)).catch(next));
+  app.get("/dists/:dist", ({params: {dist}}, res, next) => packageManeger.getDistInfo(dist).then(data => res.json(data)).catch(next));
+  app.get("/dists/:dist/source(s)?(.list)?", async (req, res, next) => {
+    const {dist} = req.params;
+    try {
+      const data = await packageManeger.getDistInfo(dist);
+      res.set("Content-Type", "text/plain");
+      const remotePath = path.posix.resolve(req.baseUrl + req.path, ".."), protocol = req.headers["x-forwarded-proto"] ?? req.protocol, hostname = process.env["RAILWAY_STATIC_URL"] ?? `${req.hostname}:${req.socket.localPort}`, host = packageConfig["apt-config"]?.sourcesHost ?? `${protocol}://${hostname}${remotePath}`;
+      return res.send(`deb ${host} ${dist} ${data.suites.join(" ")}`);
+    } catch (err) {return next(err);}
+  });
+
+  // Create Package, Package.gz and Package.xz
+  function createPackages(packages: packageSave[], dist: string, suite: string) {
+    if (!packages.length) throw new Error("Check is dist or suite have packages");
+    const reduced = packages.reduce((old, current) => {
+      if (!old[current.control.Package]) old[current.control.Package] = [];
+      old[current.control.Package].push(current);
+      return old;
+    }, {} as {[packageName: string]: packageSave[]});
+    packages = [];
+    Object.keys(reduced).forEach(packageName => {
+      reduced[packageName] = reduced[packageName].sort((b, a) => {
+        const aVersion = semver.valid(semver.coerce(a.control.Version) ?? ""), bVersion = semver.valid(semver.coerce(b.control.Version) ?? "");
+        return semver.compare(aVersion, bVersion);
+      });
+      if (!packageConfig["apt-config"]?.packagesOptions?.uniqueVersion) packages.push(...reduced[packageName]);
+      else {
+        const at = reduced[packageName].at(0);
+        if (at) packages.push(at);
+      }
+    });
+
+    let rawSize = 0, gzipSize = 0, lzmaSize = 0;
+    const mainReadstream = new Readable({read(){}}), rawSUMs = coreUtils.extendsCrypto.createHashAsync("all", mainReadstream).then(hash => ({size: rawSize, hash}));
+    const gzip = mainReadstream.pipe(createGzip()), gzipSUMs = coreUtils.extendsCrypto.createHashAsync("all", gzip).then(hash => ({size: gzipSize, hash}));
+    const lzma = mainReadstream.pipe(lzmaCompress()), lzmaSUMs = coreUtils.extendsCrypto.createHashAsync("all", lzma).then(hash => ({size: lzmaSize, hash}));
+    mainReadstream.on("data", data => rawSize += data.length);
+    gzip.on("data", data => gzipSize += data.length);
+    lzma.on("data", data => lzmaSize += data.length);
+    let fist = true;
+    for (const {control} of packages) {
+      if (!(control.Size && (control.MD5sum || control.SHA256 || control.SHA1))) continue;
+      if (fist) fist = false; else mainReadstream.push("\n\n");
+      control.Filename = `pool/${dist}/${suite}/${control.Package}/${control.Architecture}/${control.Version}/download.deb`;
+      mainReadstream.push(Object.keys(control).map(key => `${key}: ${control[key]}`).join("\n"));
+    }
+    mainReadstream.push(null);
+
+    return {
+      raw: mainReadstream,
+      gzip,
+      lzma,
+      SUMs: {
+        raw: rawSUMs,
+        gzip: gzipSUMs,
+        lzma: lzmaSUMs
+      }
+    };
+  }
+
+  app.get("/dists/(./)?:dist/:suite/binary-:arch/Packages(.(xz|gz)|)", async ({params: {dist, suite, arch}, path: reqPath}, res, next) => {
+    try {
+      const packages = createPackages(await packageManeger.getPackages(dist, suite, undefined, arch), dist, suite);
+      if (reqPath.endsWith(".gz")) return packages.gzip.pipe(res);
+      else if (reqPath.endsWith(".xz")) return packages.lzma.pipe(res);
+      else return packages.raw.pipe(res);
+    } catch (err) {return next(err);}
+  });
   // Release
-  async function createReleaseV1(dist: string) {
+  async function createRelease(dist: string) {
-    const { suites, archs } = packInfos.getDistribuition(dist);
+    if (!await packageManeger.existsDist(dist)) throw new Error("Dist exists");
-    const distConfig = repositoryConfig.repositories[dist];
+    const packagesArray = await packageManeger.getPackages(dist);
-    if (!distConfig) throw new Error("Dist not found");
+    const Release: {[key: string]: string|string[]} = {};
-    const ReleaseLines = [];
-
-    // Origin
-    const Origin = distConfig["apt-config"]?.origin ?? repositoryConfig["apt-config"]?.origin;
-    if (Origin) ReleaseLines.push(`Origin: ${Origin}`);
-
-    // Lebel
-    const Label = distConfig["apt-config"]?.label ?? repositoryConfig["apt-config"]?.label;
-    if (Label) ReleaseLines.push(`Label: ${Label}`);
-
-    // Codename if exists
-    const codename = distConfig["apt-config"]?.codename ?? repositoryConfig["apt-config"]?.codename;
-    if (codename) ReleaseLines.push(`Codename: ${codename}`);
 
     // Date
-    ReleaseLines.push(`Date: ${new Date().toUTCString()}`);
+    Release.Date = new Date().toUTCString();
 
-    // Architectures
+    // Origin
-    if (archs.length === 0) throw new Error("No architectures found");
+    const Origin = packageConfig["apt-config"]?.origin ?? packagesArray.find(x => x.aptConfig?.origin)?.aptConfig?.origin;
-    ReleaseLines.push(`Architectures: ${archs.join(" ")}`);
+    if (Origin) Release.Origin = Origin;
+
+    // Lebel
+    const Label = packageConfig["apt-config"]?.label ?? packagesArray.find(x => x.aptConfig?.label)?.aptConfig?.label;
+    if (Label) Release.Label = Label;
+
+    // Codename
+    const Codename = packageConfig["apt-config"]?.codename ?? packagesArray.find(x => x.aptConfig?.codename)?.aptConfig?.codename;
+    if (Codename) Release.Codename = Codename;
+
+    // Archs
+    const Archs = ([...(new Set(packagesArray.map(x => x.control.Architecture)))]);
+    if (!Archs.length) throw new Error("Check is dist have packages");
+    Release.Architectures = Archs.join(" ");
 
     // Components
-    if (suites.length === 0) throw new Error("No suites found");
+    const Components = ([...(new Set(packagesArray.map(x => x.suite)))]);
-    ReleaseLines.push(`Components: ${suites.join(" ")}`);
+    if (!Components.length) throw new Error("Check is dist have packages");
+    Release.Components = Components.join(" ");
 
-    const createPackagesHash = distConfig["apt-config"]?.enableHash ?? repositoryConfig["apt-config"]?.enableHash ?? true;
+    // Description
-    if (createPackagesHash) {
+
-      ReleaseLines.push("Acquire-By-Hash: no");
+    // Sum's
-      const hashs = (await Promise.all(archs.map(async arch => Promise.all(suites.map(async suite => {
+    const enableHash = Boolean(packageConfig["apt-config"]?.enableHash ?? packagesArray.find(x => x.aptConfig?.enableHash)?.aptConfig?.enableHash);
-        const [gzip, xz, raw] = await Promise.all([packInfos.createPackages({compress: "gzip", dist, arch, suite}), packInfos.createPackages({compress: "xz", dist, arch, suite}), packInfos.createPackages({dist, arch, suite})]);
+    if (enableHash) {
-        return {
+      Release.SHA256 = [];
-          gz: {
+      Release.SHA1 = [];
-            sha256: {
+      Release.MD5sum = [];
-              file: `${suite}/binary-${arch}/Packages.gz`,
+      const files = await Promise.all(Archs.map(async Arch => Promise.all(Components.map(async Component => {
-              size: gzip.size,
+        const archPackages = packagesArray.filter(x => x.control.Architecture === Arch && x.suite === Component);
-              hash: gzip.sha256
+        const {SUMs} = createPackages(archPackages, dist, Component);
-            },
+        return [
-            sha1: {
+          {
-              file: `${suite}/binary-${arch}/Packages.gz`,
+            file: `${Component}/binary-${Arch}/Packages`,
-              size: gzip.size,
+            hash: await SUMs.raw
-              hash: gzip.sha1
-            },
-            md5: {
-              file: `${suite}/binary-${arch}/Packages.gz`,
-              size: gzip.size,
-              hash: gzip.md5
-            }
           },
-          xz: {
+          {
-            sha256: {
+            file: `${Component}/binary-${Arch}/Packages.gz`,
-              file: `${suite}/binary-${arch}/Packages.xz`,
+            hash: await SUMs.gzip
-              size: xz.size,
-              hash: xz.sha256
-            },
-            sha1: {
-              file: `${suite}/binary-${arch}/Packages.xz`,
-              size: xz.size,
-              hash: xz.sha1
-            },
-            md5: {
-              file: `${suite}/binary-${arch}/Packages.xz`,
-              size: xz.size,
-              hash: xz.md5
-            }
           },
-          raw: {
+          {
-            sha256: {
+            file: `${Component}/binary-${Arch}/Packages.xz`,
-              file: `${suite}/binary-${arch}/Packages`,
+            hash: await SUMs.lzma
-              size: raw.size,
-              hash: raw.sha256
-            },
-            sha1: {
-              file: `${suite}/binary-${arch}/Packages`,
-              size: raw.size,
-              hash: raw.sha1
-            },
-            md5: {
-              file: `${suite}/binary-${arch}/Packages`,
-              size: raw.size,
-              hash: raw.md5
-            }
           }
-        };
+        ]
-      }))))).flat(2);
+      })))).then(f => f.flat(3));
 
-      const sha256 = hashs.map(hash => hash.raw.sha256).concat(hashs.map(hash => hash.gz.sha256)).concat(hashs.map(hash => hash.xz.sha256));
+      files.forEach(({file, hash}) => {
-      if (sha256.length > 0) ReleaseLines.push(`SHA256:${sha256.sort().map((hash) => `\n  ${hash.hash} ${hash.size} ${hash.file}`).join("")}`);
+        if (hash.hash.sha256) (Release.SHA256 as string[]).push(`${hash.hash.sha256} ${hash.size} ${file}`);
-
+        if (hash.hash.sha1) (Release.SHA1 as string[]).push(`${hash.hash.sha1} ${hash.size} ${file}`);
-      const sha1 = hashs.map(hash => hash.raw.sha1).concat(hashs.map(hash => hash.gz.sha1)).concat(hashs.map(hash => hash.xz.sha1));
+        if (hash.hash.md5) (Release.MD5sum as string[]).push(`${hash.hash.md5} ${hash.size} ${file}`);
-      if (sha1.length > 0) ReleaseLines.push(`SHA1:${sha1.sort().map((hash) => `\n  ${hash.hash} ${hash.size} ${hash.file}`).join("")}`);
+      });
-
-      const md5 = hashs.map(hash => hash.raw.md5).concat(hashs.map(hash => hash.gz.md5)).concat(hashs.map(hash => hash.xz.md5));
-      if (md5.length > 0) ReleaseLines.push(`MD5Sum:${md5.sort().map((hash) => `\n  ${hash.hash} ${hash.size} ${hash.file}`).join("")}`);
     }
 
-    return ReleaseLines.join("\n");
+    return Object.keys(Release).reduce((old, key) => {
+      if (Array.isArray(Release[key])) old.push(`${key}:\n  ${(Release[key] as string[]).join("\n  ")}`);
+      else old.push(`${key}: ${Release[key]}`);
+      return old;
+    }, []).join("\n");
   }
-  app.get("/dists/(./)?:dist/Release", (req, res, next) => createReleaseV1(req.params.dist).then((data) => res.setHeader("Content-Type", "text/plain").send(data)).catch(next));
+  app.get("/dists/(./)?:dist/Release", ({params: {dist}}, res, next) => createRelease(dist).then(release => res.setHeader("Content-Type", "text/plain").send(release)).catch(next));
-  app.get("/dists/(./)?:dist/InRelease", (req, res, next) => {
+
-    const Key = repositoryConfig["apt-config"]?.pgpKey;
+  const pgpKey = packageConfig["apt-config"]?.pgpKey;
-    if (!Key) return res.status(404).json({error: "No PGP key found"});
+  app.get("/dists/(./)?:dist/inRelease", async (req, res, next) => {
+    if (!pgpKey) return res.status(404).json({error: "No PGP key found"});
     return Promise.resolve().then(async () => {
-      const privateKey = Key.passphrase ? await openpgp.decryptKey({privateKey: await openpgp.readPrivateKey({ armoredKey: Key.private }), passphrase: Key.passphrase}) : await openpgp.readPrivateKey({ armoredKey: Key.private });
+      const privateKey = pgpKey.passphrase ? await openpgp.decryptKey({privateKey: await openpgp.readPrivateKey({ armoredKey: pgpKey.private }), passphrase: pgpKey.passphrase}) : await openpgp.readPrivateKey({ armoredKey: pgpKey.private });
-      const Release = await createReleaseV1(req.params.dist);
+      const Release = await createRelease(req.params.dist);
       return res.setHeader("Content-Type", "text/plain").send(await openpgp.sign({
         signingKeys: privateKey,
         format: "armored",
@@ -213,238 +238,27 @@ export default async function main(configPath: string) {
       }));
     }).catch(next);
   });
-  app.get("/dists/(./)?:dist/Release.gpg", (req, res, next) => {
+  app.get("/dists/(./)?:dist/Release.gpg", async (req, res, next) => {
-    const Key = repositoryConfig["apt-config"]?.pgpKey;
+    if (!pgpKey) return res.status(404).json({error: "No PGP key found"});
-    if (!Key) return res.status(404).json({error: "No PGP key found"});
     return Promise.resolve().then(async () => {
-      const privateKey = Key.passphrase ? await openpgp.decryptKey({privateKey: await openpgp.readPrivateKey({ armoredKey: Key.private }), passphrase: Key.passphrase}) : await openpgp.readPrivateKey({ armoredKey: Key.private });
+      const privateKey = pgpKey.passphrase ? await openpgp.decryptKey({privateKey: await openpgp.readPrivateKey({ armoredKey: pgpKey.private }), passphrase: pgpKey.passphrase}) : await openpgp.readPrivateKey({ armoredKey: pgpKey.private });
-      const Release = await createReleaseV1(req.params.dist);
+      const Release = await createRelease(req.params.dist);
       return res.setHeader("Content-Type", "text/plain").send(await openpgp.sign({
         signingKeys: privateKey,
         message: await openpgp.createMessage({text: Release}),
       }));
     }).catch(next);
   });
-
+  // Public key
-  // 404 handler
+  if (pgpKey) app.get(["/public_key", "/public.gpg"], async ({res}) => {
-  app.use((_req, res) => {
+    if (!pgpKey) return res.status(400).json({error: "This repository no sign Packages files"});
-    res.status(404).json({error: "Not found"});
+    const pubKey = (await openpgp.readKey({ armoredKey: pgpKey.public })).armor();
+    return res.setHeader("Content-Type", "application/pgp-keys").send(pubKey);
   });
 
-  // Error handler
+  return {
-  app.use((err, _req, res, _next) => {
+    app,
-    console.error(err);
+    packageManeger,
-    res.status(500).json({
+    packageConfig
-      error: err?.message||err,
+  };
-      stack: err?.stack?.split("\n"),
+}
-    });
-  });
-
-  // Listen HTTP server
-  const port = process.env.PORT ?? repositoryConfig["apt-config"].portListen ?? 0;
-  app.listen(port, function () {return console.log(`apt-repo listening at http://localhost:${this.address().port}`);});
-
-  // Loading and update packages
-  let cronJobs: CronJob[] = [];
-  const waitPromises: Promise<void>[] = [];
-  const saveFile = repositoryConfig["apt-config"]?.saveFiles;
-  const rootPool = repositoryConfig["apt-config"]?.poolPath;
-  for (const dist in repositoryConfig.repositories) {
-    const targets = repositoryConfig.repositories[dist].targets;
-    for (const repository of targets) {
-      const update = async () => {
-        if (repository.from === "mirror") {
-          return Promise.all(Object.keys(repository.dists).map(async distName => {
-            const distInfo = repository.dists[distName];
-            const packagesData = distInfo.suites ? await Promise.all(distInfo.suites.map(async suite => getPackages(repository.uri, {dist: distName, suite}))).then(U => U.flat()) : await getPackages(repository.uri, {dist: distName});
-            return packagesData.forEach(({Package: control}) => {
-              const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
-              const getStream = async () => {
-                if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
-                if (saveFile) {
-                  const mainPath = path.resolve(filePool, "..");
-                  if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
-                  const fileStream = await httpRequest.pipeFetch(control.Filename);
-                  fileStream.pipe(createWriteStream(filePool));
-                  return fileStream;
-                }
-                return httpRequest.pipeFetch(control.Filename);
-              }
-              return packInfos.addPackage(dist, repository.suite ?? "main", {repositoryConfig: repository, control, getStream});
-            });
-          }));
-        } else if (repository.from === "oci") {
-          const registry = await DockerRegistry.Manifest.Manifest(repository.image, repository.platfom_target);
-          return registry.layersStream((data) => {
-            if (!(["gzip", "gz", "tar"]).some(ends => data.layer.mediaType.endsWith(ends))) return data.next();
-            data.stream.pipe(tar.list({
-              async onentry(entry) {
-                if (!entry.path.endsWith(".deb")) return null;
-                const control = await DebianPackage.extractControl(entry as any);
-                const suite = repository.suite ?? "main";
-                packInfos.addPackage(dist, suite, {
-                  repositoryConfig: repository,
-                  control,
-                  async getStream() {
-                    const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
-                    if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
-                    return new Promise<Readable>((done, reject) => registry.blobLayerStream(data.layer.digest).then(stream => {
-                      stream.on("error", reject);
-                      stream.pipe(tar.list({
-                        async onentry(getEntry) {
-                          if (getEntry.path !== entry.path) return null;
-                          if (saveFile) {
-                            const mainPath = path.resolve(filePool, "..");
-                            if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
-                            entry.pipe(createWriteStream(filePool));
-                          }
-                          return done(getEntry as any);
-                        }
-                      // @ts-ignore
-                      }).on("error", reject));
-                    }).catch(reject));
-                  }
-                });
-              }
-            }));
-          });
-        } else if (repository.from === "github_release") {
-          if (repository.tags) {
-            const release = await Promise.all(repository.tags.map(async releaseTag => httpRequestGithub.getRelease({
-              owner: repository.owner,
-              repository: repository.repository,
-              token: repository.token,
-              releaseTag,
-            })));
-            return Promise.all(release.map(async release => Promise.all(release.assets.map(async ({browser_download_url, name}) => {
-              if (!name.endsWith(".deb")) return null;
-              const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(browser_download_url));
-              const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
-              const getStream = async () => {
-                if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
-                if (saveFile) {
-                  const mainPath = path.resolve(filePool, "..");
-                  if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
-                  const fileStream = await httpRequest.pipeFetch(browser_download_url);
-                  fileStream.pipe(createWriteStream(filePool));
-                  return fileStream;
-                }
-                return httpRequest.pipeFetch(browser_download_url);
-              }
-              return packInfos.addPackage(dist, repository.suite ?? release.tag_name, {repositoryConfig: repository, control, getStream});
-            })))).then(data => data.flat(2).filter(Boolean));
-          }
-          const release = await httpRequestGithub.getRelease({owner: repository.owner, repository: repository.repository, token: repository.token, peer: repository.assetsLimit, all: false});
-          return Promise.all(release.map(async release => Promise.all(release.assets.map(async ({browser_download_url, name}) => {
-            if (!name.endsWith(".deb")) return null;
-            const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(browser_download_url));
-            const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
-            const getStream = async () => {
-              if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
-              if (saveFile) {
-                const mainPath = path.resolve(filePool, "..");
-                if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
-                const fileStream = await httpRequest.pipeFetch(browser_download_url);
-                fileStream.pipe(createWriteStream(filePool));
-                return fileStream;
-              }
-              return httpRequest.pipeFetch(browser_download_url);
-            }
-            return packInfos.addPackage(dist, repository.suite ?? release.tag_name, {repositoryConfig: repository, control, getStream});
-          })))).then(data => data.flat(2).filter(Boolean));
-        } else if (repository.from === "github_tree") {
-          const { tree } = await httpRequestGithub.githubTree(repository.owner, repository.repository, repository.tree);
-          const filtedTree = tree.filter(({path: remotePath}) => {
-            if (repository.path) return repository.path.some(repoPath => {
-              if (!remotePath.startsWith("/")) remotePath = "/" + remotePath;
-              if (typeof repoPath === "string") {
-                if (!repoPath.startsWith("/")) repoPath = "/" + repoPath;
-                return remotePath.startsWith(repoPath);
-              }
-              return false;
-            });
-            return true;
-          }).filter(({path, type}) => path.endsWith(".deb") && type === "blob");
-          return Promise.all(filtedTree.map(async ({path: filePath}) => {
-            const downloadUrl = `https://raw.githubusercontent.com/${repository.owner}/${repository.repository}/${repository.tree}/${filePath}`;
-            const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(downloadUrl));
-            const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
-            const getStream = async () => {
-              if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
-              if (saveFile) {
-                const mainPath = path.resolve(filePool, "..");
-                if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
-                const fileStream = await httpRequest.pipeFetch(downloadUrl);
-                fileStream.pipe(createWriteStream(filePool));
-                return fileStream;
-              }
-              return httpRequest.pipeFetch(downloadUrl);
-            }
-            return packInfos.addPackage(dist, repository.suite ?? "main", {repositoryConfig: repository, control, getStream});
-          }));
-        } else if (repository.from === "google_drive") {
-          const client_id = repository.appSettings.client_id;
-          const client_secret = repository.appSettings.client_secret;
-          const token = repository.appSettings.token;
-          const googleDriver = await coreUtils.googleDriver.GoogleDriver(client_id, client_secret, {
-            token,
-            async authCallback(url, token) {
-              if (url) console.log("Please visit this url to auth google driver: %s", url);
-              else console.log("Google driver auth success, please save token to config file, token: %s", token);
-            },
-          });
-          const files = (repository.folderId ? (await Promise.all(repository.folderId.map(async folderId => await googleDriver.listFiles(folderId)))).flat() : await googleDriver.listFiles());
-          return Promise.all(files.filter(({name, isTrashedFile}) => !isTrashedFile && name.endsWith(".deb")).map(async fileData => {
-            const control = await DebianPackage.extractControl(await googleDriver.getFileStream(fileData.id));
-            const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
-            const getStream = async () => {
-              if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
-              if (saveFile) {
-                const mainPath = path.resolve(filePool, "..");
-                if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
-                const fileStream = await googleDriver.getFileStream(fileData.id);
-                fileStream.pipe(createWriteStream(filePool));
-                return fileStream;
-              }
-              return googleDriver.getFileStream(fileData.id);
-            }
-            return packInfos.addPackage(dist, repository.suite ?? "main", {repositoryConfig: repository, control, getStream});
-          }));
-        } else if (repository.from === "oracle_bucket") {
-          const oracleBucket = await coreUtils.oracleBucket(repository.region as any, repository.bucketName, repository.bucketNamespace, repository.auth);
-          return Promise.all((await oracleBucket.fileList()).filter(({name}) => name.endsWith(".deb")).map(async fileData => {
-            const control = await DebianPackage.extractControl(await oracleBucket.getFileStream(fileData.name));
-            const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
-            const getStream = async () => {
-              if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
-              if (saveFile) {
-                const mainPath = path.resolve(filePool, "..");
-                if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
-                const fileStream = await oracleBucket.getFileStream(fileData.name);
-                fileStream.pipe(createWriteStream(filePool));
-                return fileStream;
-              }
-              return oracleBucket.getFileStream(fileData.name);
-            }
-            return packInfos.addPackage(dist, repository.suite ?? "main", {repositoryConfig: repository, control, getStream});
-          }));
-        }
-        return null;
-      }
-      waitPromises.push(update().then(() => {
-        const cron = (repository.cronRefresh ?? []).map((cron) => new CronJob(cron, update));
-        cron.forEach((cron) => cron.start());
-        cronJobs.push(...cron);
-      }).catch(console.error));
-    }
-  }
-  // watch config file changes
-  watchFile(configPath, async () => {
-    console.info("Config file changed, reloading config and update packages...");
-    repositoryConfig = await getConfig(configPath);
-    cronJobs.forEach((cron) => cron.stop());
-    cronJobs = [];
-  });
-  // await Promise.all(waitPromises);
-  return app;
-}

src/index.ts

@@ -6,6 +6,8 @@ import yargs from "yargs";
 import path from "node:path";
 import repo from "./express_route.js";
 import yaml from "yaml";
+import os from "node:os";
+import cluster from "node:cluster";
 
 yargs(process.argv.slice(2)).version(false).help().demandCommand().strictCommands().alias("h", "help").option("cofig-path", {
   type: "string",
@@ -84,12 +86,69 @@ yargs(process.argv.slice(2)).version(false).help().demandCommand().strictCommand
     }).parseSync();
     const config = await getConfig(options.cofigPath);
     const base64 = Buffer.from(options.json ? JSON.stringify(config) : yaml.stringify(config)).toString("base64");
-    if (options.output) return writeFile(options.output, base64).then(() => console.log("Saved to '%s'", options.output));
+    if (options.output) return writeFile(options.output, "base64:"+base64).then(() => console.log("Saved to '%s'", options.output));
     console.log("base64:%s", base64);
   });
-}).command("server", "Run HTTP serber", yargs => {
+}).command("server", "Run HTTP serber", async yargs => {
-  const options = yargs.parseSync();
+  const options = yargs.option("cpus", {
-  const envs = Object.keys(process.env).filter(key => key.startsWith("APT_STREAM"));
+    type: "number",
-  if (envs.length > 0) return Promise.all(envs.map(async env => repo(`env:${env}`)));
+    default: os.cpus().length/2,
-  return repo(options.cofigPath);
+    alias: "C",
+    demandOption: false,
+    description: "Number of cpus to use in Cluster"
+  }).parseSync();
+  console.log("Starting server...");
+  process.on("unhandledRejection", err => console.error(err));
+  process.on("uncaughtException", err => console.error(err));
+  const { app, packageConfig, packageManeger } = await repo(Object.keys(process.env).find(key => key.startsWith("APT_STREAM")) ? `env:${Object.keys(process.env).find(key => key.startsWith("APT_STREAM"))}` : options.cofigPath);
+  app.all("*", ({res}) => res.status(404).json({
+    error: "Endpoint not exists",
+    message: "Endpoint not exists, check the documentation for more information"
+  }));
+  app.use((err, {}, res, {}) => {
+    console.error(err);
+    const stack: string = err?.stack ?? "No stack";
+    res.status(400).json({
+      error: "Internal Server Error",
+      message: "There was an error on our part, sorry for the inconvenience",
+      stack: {
+        forUser: "Create issue in apt-stream repository (https://github.com/Sirherobrine23/apt-stream/issues) with value of 'forDeveloper'",
+        forDeveloper: stack
+      },
+    });
+  });
+  const port = process.env.PORT ?? packageConfig["apt-config"]?.portListen ?? 3000;
+  if (options.cpus > 1) {
+    if (cluster.isWorker) {
+      console.log("Worker %d running, PID: %f", cluster.worker?.id ?? "No ID", process.pid);
+      app.listen(port, function() {
+        console.log("Work apt Stream Port listen on %f", this.address()?.port);
+      });
+      return;
+    }
+    console.log("Work master, PID %f, starting workers ...", process.pid);
+    for (let i = 0; i < options.cpus; i++) cluster.fork({...process.env, workNumber: i});
+    cluster.on("error", err => {
+      console.log(err?.stack ?? String(err));
+      // process.exit(1);
+    }).on("exit", (worker, code, signal: NodeJS.Signals) => {
+      // if (process[Symbol.for("ts-node.register.instance")]) cluster.setupPrimary({/* Fix for ts-node */ execArgv: ["--loader", "ts-node/esm"]});
+      if (signal === "SIGKILL") return console.log("Worker %d was killed", worker?.id ?? "No ID");
+      else if (signal === "SIGABRT") return console.log("Worker %d was aborted", worker?.id ?? "No ID");
+      else if (signal === "SIGTERM") return console.log("Worker %d was terminated", worker?.id ?? "No ID");
+      console.log("Worker %d died with code: %s, Signal: %s", worker?.id ?? "No ID", code, signal ?? "No Signal");
+      cluster.fork();
+    }).on("online", worker => console.log("Worker %d is online", worker?.id ?? "No ID"));
+  } else {
+    console.warn("Running without cluster, this is not recommended for production");
+    app.listen(port, function() {console.log("Apt Stream Port listen on %f", this.address()?.port)});
+  }
+
+  for (const distName in packageConfig.repositories) {
+    const dist = packageConfig.repositories[distName];
+    for (const target of dist.targets) {
+      await packageManeger.loadRepository(distName, target, packageConfig["apt-config"], packageConfig).catch(err => console.error(String(err)));
+      console.log("Complete load repository '%s'", distName);
+    }
+  }
 }).parseAsync();

src/mirror.ts

@@ -82,7 +82,6 @@ export async function getPackages(uri: string, options: {dist: string, suite?: s
   const hashs: {file: string, Package: DebianPackage.debianControl}[] = [];
   async function addPackages(dist: string, file: string, fn?: (data: DebianPackage.debianControl) => void) {
     const urlRequest = `${uri}/dists/${dist}/${file}`;
-    console.log(`Requesting ${urlRequest}`);
     await new Promise<void>(async (done, reject) => {
       const stream = (urlRequest.endsWith(".gz")||urlRequest.endsWith(".xz")) ? (await httpRequest.pipeFetch(urlRequest)).pipe(urlRequest.endsWith(".gz") ? zlib.createGunzip() : lzmaDecompressor()) : await httpRequest.pipeFetch(urlRequest);
       stream.on("error", (err: any) => {
@@ -92,14 +91,14 @@ export async function getPackages(uri: string, options: {dist: string, suite?: s
           reject(err);
         }
       });
-      let data = "";
+      let data: string;
       stream.pipe(new Writable({
         final(callback) {
           done();
           callback();
         },
         write(chunkR, encoding, callback) {
-          data = data + (encoding === "binary" ? chunkR.toString("utf8") : Buffer.from(chunkR).toString("utf8"));
+          data = (data ?? "") + (encoding === "binary" ? chunkR.toString("utf8") : Buffer.from(chunkR).toString("utf8"));
           data.split(/^\n/).forEach((v) => {
             if (v.trim()) {
               data = data.replace(v, "");

src/packagesData.ts

@@ -0,0 +1,447 @@
+import coreUtils, { DebianPackage, DockerRegistry, extendFs, httpRequest, httpRequestGithub } from "@sirherobrine23/coreutils";
+import { createReadStream, createWriteStream, promises as fs } from "node:fs";
+import { MongoClient, ServerApiVersion, Filter } from "mongodb";
+import { apt_config, backendConfig, repository } from "./repoConfig.js";
+import { getPackages as mirror } from "./mirror.js";
+import { Readable } from "node:stream";
+import { format } from "node:util";
+import cluster from "node:cluster";
+import path from "node:path";
+import tar from "tar";
+
+export type packageSave = {
+  dist: string,
+  suite: string,
+  repository: repository,
+  aptConfig?: apt_config,
+  control: DebianPackage.debianControl,
+  restoreFileStream?: {
+    from: repository["from"],
+    [key: string]: any,
+  },
+  getFileStream: () => Promise<Readable>,
+};
+
+export type packageManegerV2 = {
+  loadRepository: (distName: string, repo: repository, packageAptConfig?: apt_config, aptConfig?: backendConfig) => Promise<any>,
+  getPackages: (dist?: string, suite?: string, Package?: string, Arch?: string, Version?: string) => Promise<packageSave[]>,
+  deletePackage: (repo: Partial<packageSave>) => Promise<packageSave>,
+  addPackage: (repo: packageSave) => Promise<void>,
+  existsDist: (dist: string) => Promise<boolean>,
+  existsSuite: (dist: string, suite: string) => Promise<boolean>,
+  getDists: () => Promise<string[]>,
+  getDistInfo: (dist: string) => Promise<{
+    packagesCount: number,
+    arch: string[],
+    packagesName: string[],
+    suites: string[],
+  }>,
+};
+
+/**
+ * Maneger and Load packages to Database or internal object (Nodejs Heap Memory, if large data use Database)
+ * @returns
+ */
+export default async function packageManeger(config: backendConfig): Promise<packageManegerV2> {
+  const partialConfig: Partial<packageManegerV2> = {};
+  partialConfig.getDists = async function getDists() {
+    const packages = await partialConfig.getPackages();
+    return [...new Set(packages.map(U => U.dist))];
+  }
+
+  partialConfig.getDistInfo = async function getDistInfo(dist: string) {
+    const packages = await partialConfig.getPackages(dist);
+    return {
+      packagesCount: packages.length,
+      packagesName: [...new Set(packages.map(U => U.control.Package))],
+      arch: [...new Set(packages.map(U => U.control.Architecture))],
+      suites: [...new Set(packages.map(U => U.suite))],
+    };
+  }
+
+  partialConfig.loadRepository = async function loadRepository(distName: string, repository: repository, packageAptConfig?: apt_config, aptConfig?: backendConfig) {
+    const saveFile = aptConfig["apt-config"]?.saveFiles ?? false;
+    const rootPool = aptConfig["apt-config"]?.poolPath ?? path.join(process.cwd(), "pool");
+    if (repository.from === "mirror") {
+      // Ingore fast load data for low ram memory
+      for (const repoDistName in repository.dists) {
+        const distInfo = repository.dists[repoDistName];
+        const packagesData: Awaited<ReturnType<typeof mirror>> = [];
+        if (!distInfo.suites) await mirror(repository.uri, {dist: distName}).then(U => packagesData.push(...U));
+        else for (const suite of distInfo.suites) await mirror(repository.uri, {dist: repoDistName, suite}).then(U => packagesData.push(...U));
+        const partialPromises = packagesData.map(({Package: control}) => {
+          const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+          const getStream = async () => {
+            if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+            if (saveFile) {
+              const mainPath = path.resolve(filePool, "..");
+              if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
+              const fileStream = await httpRequest.pipeFetch(control.Filename);
+              fileStream.pipe(createWriteStream(filePool));
+              return fileStream;
+            }
+            return httpRequest.pipeFetch(control.Filename);
+          }
+          return partialConfig.addPackage({
+            dist: distName,
+            suite: repository.suite ?? "main",
+            repository: repository,
+            control,
+            aptConfig: packageAptConfig ?? aptConfig["apt-config"],
+            getFileStream: getStream,
+            restoreFileStream: {
+              from: "mirror",
+              fileUrl: control.Filename,
+            }
+          }).catch(err => err);
+        });
+
+        return Promise.all(partialPromises);
+      }
+    } else if (repository.from === "oci") {
+      const registry = await DockerRegistry.Manifest.Manifest(repository.image, repository.platfom_target);
+      return registry.layersStream((data) => {
+        if (!(["gzip", "gz", "tar"]).some(ends => data.layer.mediaType.endsWith(ends))) return data.next();
+        data.stream.pipe(tar.list({
+          async onentry(entry) {
+            if (!entry.path.endsWith(".deb")) return null;
+            const control = await DebianPackage.extractControl(entry as any);
+            const suite = repository.suite ?? "main";
+            async function getStream() {
+              const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+              if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+              return new Promise<Readable>((done, reject) => registry.blobLayerStream(data.layer.digest).then(stream => {
+                stream.on("error", reject);
+                stream.pipe(tar.list({
+                  async onentry(getEntry) {
+                    if (getEntry.path !== entry.path) return null;
+                    if (saveFile) {
+                      const mainPath = path.resolve(filePool, "..");
+                      if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
+                      entry.pipe(createWriteStream(filePool));
+                    }
+                    return done(getEntry as any);
+                  }
+                // @ts-ignore
+                }).on("error", reject));
+              }).catch(reject));
+            }
+            return partialConfig.addPackage({
+              dist: distName,
+              suite,
+              repository: repository,
+              control,
+              aptConfig: packageAptConfig ?? aptConfig["apt-config"],
+              getFileStream: getStream,
+              restoreFileStream: {
+                from: "oci",
+                digest: data.layer.digest,
+                path: entry.path,
+              }
+            });
+          }
+        }));
+      });
+    } else if (repository.from === "github_release") {
+      if (repository.tags) {
+        const release = await Promise.all(repository.tags.map(async releaseTag => httpRequestGithub.getRelease({
+          owner: repository.owner,
+          repository: repository.repository,
+          token: repository.token,
+          releaseTag,
+        })));
+        return Promise.all(release.map(async release => Promise.all(release.assets.map(async ({browser_download_url, name}) => {
+          if (!name.endsWith(".deb")) return null;
+          const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(browser_download_url));
+          const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+          const getStream = async () => {
+            if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+            if (saveFile) {
+              const mainPath = path.resolve(filePool, "..");
+              if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
+              const fileStream = await httpRequest.pipeFetch(browser_download_url);
+              fileStream.pipe(createWriteStream(filePool));
+              return fileStream;
+            }
+            return httpRequest.pipeFetch(browser_download_url);
+          }
+          return partialConfig.addPackage({
+            dist: distName,
+            suite: repository.suite ?? "main",
+            repository: repository,
+            control,
+            aptConfig: packageAptConfig ?? aptConfig["apt-config"],
+            getFileStream: getStream,
+            restoreFileStream: {
+              from: "github_release",
+              fileUrl: browser_download_url,
+            }
+          }).catch(err => {});
+        })))).then(data => data.flat(2).filter(Boolean));
+      }
+      const release = await httpRequestGithub.getRelease({owner: repository.owner, repository: repository.repository, token: repository.token, peer: repository.assetsLimit, all: false});
+      return Promise.all(release.map(async release => Promise.all(release.assets.map(async ({browser_download_url, name}) => {
+        if (!name.endsWith(".deb")) return null;
+        const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(browser_download_url));
+        const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+        const getStream = async () => {
+          if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+          if (saveFile) {
+            const mainPath = path.resolve(filePool, "..");
+            if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
+            const fileStream = await httpRequest.pipeFetch(browser_download_url);
+            fileStream.pipe(createWriteStream(filePool));
+            return fileStream;
+          }
+          return httpRequest.pipeFetch(browser_download_url);
+        }
+        return partialConfig.addPackage({
+          dist: distName,
+          suite: repository.suite ?? "main",
+          repository: repository,
+          control,
+          aptConfig: packageAptConfig ?? aptConfig["apt-config"],
+          getFileStream: getStream,
+          restoreFileStream: {
+            from: "github_release",
+            fileUrl: browser_download_url,
+          }
+        }).catch(err => {});
+      })))).then(data => data.flat(2).filter(Boolean));
+    } else if (repository.from === "github_tree") {
+      const { tree } = await httpRequestGithub.githubTree(repository.owner, repository.repository, repository.tree);
+      const filtedTree = tree.filter(({path: remotePath}) => {
+        if (repository.path) return repository.path.some(repoPath => {
+          if (!remotePath.startsWith("/")) remotePath = "/" + remotePath;
+          if (typeof repoPath === "string") {
+            if (!repoPath.startsWith("/")) repoPath = "/" + repoPath;
+            return remotePath.startsWith(repoPath);
+          }
+          return false;
+        });
+        return true;
+      }).filter(({path, type}) => path.endsWith(".deb") && type === "blob");
+      return Promise.all(filtedTree.map(async ({path: filePath}) => {
+        const downloadUrl = `https://raw.githubusercontent.com/${repository.owner}/${repository.repository}/${repository.tree}/${filePath}`;
+        const control = await DebianPackage.extractControl(await httpRequest.pipeFetch(downloadUrl));
+        const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+        const getStream = async () => {
+          if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+          if (saveFile) {
+            const mainPath = path.resolve(filePool, "..");
+            if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
+            const fileStream = await httpRequest.pipeFetch(downloadUrl);
+            fileStream.pipe(createWriteStream(filePool));
+            return fileStream;
+          }
+          return httpRequest.pipeFetch(downloadUrl);
+        }
+        return partialConfig.addPackage({
+          dist: distName,
+          suite: repository.suite ?? "main",
+          repository: repository,
+          control,
+          aptConfig: packageAptConfig ?? aptConfig["apt-config"],
+          getFileStream: getStream,
+          restoreFileStream: {
+            from: "github_tree",
+            fileUrl: downloadUrl,
+          }
+        }).catch(err => {});
+      }));
+    } else if (repository.from === "google_drive") {
+      const client_id = repository.appSettings.client_id;
+      const client_secret = repository.appSettings.client_secret;
+      const token = repository.appSettings.token;
+      const googleDriver = await coreUtils.googleDriver.GoogleDriver(client_id, client_secret, {
+        token,
+        async authCallback(url, token) {
+          if (url) console.log("Please visit this url to auth google driver: %s", url);
+          else console.log("Google driver auth success, please save token to config file, token: %s", token);
+        },
+      });
+      const files = (repository.folderId ? (await Promise.all(repository.folderId.map(async folderId => await googleDriver.listFiles(folderId)))).flat() : await googleDriver.listFiles());
+      return Promise.all(files.filter(({name, isTrashedFile}) => !isTrashedFile && name.endsWith(".deb")).map(async fileData => {
+        const control = await DebianPackage.extractControl(await googleDriver.getFileStream(fileData.id));
+        const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+        const getStream = async () => {
+          if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+          if (saveFile) {
+            const mainPath = path.resolve(filePool, "..");
+            if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
+            const fileStream = await googleDriver.getFileStream(fileData.id);
+            fileStream.pipe(createWriteStream(filePool));
+            return fileStream;
+          }
+          return googleDriver.getFileStream(fileData.id);
+        }
+        return partialConfig.addPackage({
+          dist: distName,
+          suite: repository.suite ?? "main",
+          repository: repository,
+          control,
+          aptConfig: packageAptConfig ?? aptConfig["apt-config"],
+          getFileStream: getStream,
+          restoreFileStream: {
+            from: "google_drive",
+            fileId: fileData.id,
+          }
+        }).catch(err => {});
+      }));
+    } else if (repository.from === "oracle_bucket") {
+      const oracleBucket = await coreUtils.oracleBucket(repository.region as any, repository.bucketName, repository.bucketNamespace, repository.auth);
+      return Promise.all((await oracleBucket.fileList()).filter(({name}) => name.endsWith(".deb")).map(async fileData => {
+        const control = await DebianPackage.extractControl(await oracleBucket.getFileStream(fileData.name));
+        const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+        const getStream = async () => {
+          if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+          if (saveFile) {
+            const mainPath = path.resolve(filePool, "..");
+            if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
+            const fileStream = await oracleBucket.getFileStream(fileData.name);
+            fileStream.pipe(createWriteStream(filePool));
+            return fileStream;
+          }
+          return oracleBucket.getFileStream(fileData.name);
+        }
+        return partialConfig.addPackage({
+          dist: distName,
+          suite: repository.suite ?? "main",
+          repository: repository,
+          control,
+          aptConfig: packageAptConfig ?? aptConfig["apt-config"],
+          getFileStream: getStream,
+          restoreFileStream: {
+            from: "oracle_bucket",
+            fileName: fileData.name,
+          }
+        }).catch(err => {});
+      }));
+    }
+
+    throw new Error(`Unknown repository from: ${(repository as any)?.from ?? "undefined"}`);
+  }
+
+  if (config["apt-config"]?.mongodb) {
+    // Connect to database
+    const mongoConfig = config["apt-config"].mongodb;
+    const mongoClient = await (new MongoClient(mongoConfig.uri, {serverApi: ServerApiVersion.v1})).connect();
+    const collection = mongoClient.db(mongoConfig.db ?? "aptStream").collection<packageSave>(mongoConfig.collection ?? "packagesData");
+
+    // Drop collection
+    if (cluster.isPrimary) {
+      if (mongoConfig.dropCollention && await collection.findOne()) {
+        await collection.drop();
+        console.log("Drop collection: %s", mongoConfig.collection ?? "packagesData");
+      }
+    }
+
+    // Add package to database
+    partialConfig.addPackage = async function addPackage(repo) {
+      const existsPackage = await collection.findOne({dist: repo.dist, suite: repo.suite, "control.Package": repo.control.Package, "control.Version": repo.control.Version, "control.Architecture": repo.control.Architecture});
+      if (existsPackage) throw new Error(format("Package (%s/%s: %s) already exists!", repo.control.Package, repo.control.Version, repo.control.Architecture));
+      await collection.insertOne(repo);
+      console.log("Added '%s', version: %s, Arch: %s, in to %s/%s", repo.control.Package, repo.control.Version, repo.control.Architecture, repo.dist, repo.suite);
+    }
+
+    // Delete package
+    partialConfig.deletePackage = async function deletePackage(repo) {
+      const packageDelete = (await collection.findOneAndDelete({dist: repo.dist, suite: repo.suite, "control.Package": repo.control.Package, "control.Version": repo.control.Version, "control.Architecture": repo.control.Architecture}))?.value;
+      if (!packageDelete) throw new Error("Package not found!");
+      console.info("Deleted '%s', version: %s, Arch: %s, from %s/%s", packageDelete.control.Package, packageDelete.control.Version, packageDelete.control.Architecture, packageDelete.dist, packageDelete.suite);
+      return packageDelete;
+    }
+
+    // Exists
+    partialConfig.existsDist = async function existsDist(dist) {
+      return (await collection.findOne({dist})) ? true : false;
+    }
+    partialConfig.existsSuite = async function existsSuite(dist, suite) {
+      if (await partialConfig.existsDist(dist)) return (await collection.findOne({dist, suite})) ? true : false;
+      return false;
+    }
+
+    // Packages
+    function fixPackage(data: packageSave): packageSave {
+      if (!data.restoreFileStream) throw new Error("cannot restore file stream!");
+      data.getFileStream = async function getFileStream() {
+        if (data.restoreFileStream.fileUrl) return coreUtils.httpRequest.pipeFetch(data.restoreFileStream.fileUrl);
+        if (data.restoreFileStream.from === "google_drive" && data.repository.from === "google_drive") {
+          const { appSettings } = data.repository;
+          const googleDrive = await coreUtils.googleDriver.GoogleDriver(appSettings.client_id, appSettings.client_secret, {token: appSettings.token});
+          return googleDrive.getFileStream(data.restoreFileStream.fileId);
+        } else if (data.restoreFileStream.from === "oci" && data.repository.from === "oci") {
+          const oci = await coreUtils.DockerRegistry(data.repository.image);
+          return new Promise((done, reject) => {
+            oci.blobLayerStream(data.restoreFileStream.digest).then((stream) => {
+              stream.pipe(tar.list({
+                filter: (path) => path === data.restoreFileStream.fileName,
+                onentry: (entry) => done(entry as any)
+              }))
+            }).catch(reject);
+          });
+        }
+        throw new Error("Cannot restore file stream!");
+      }
+      return data;
+    }
+    partialConfig.getPackages = async function getPackages(dist, suite, Package, Arch, Version) {
+      const doc: Filter<packageSave> = {};
+      if (dist) {
+        if (!await partialConfig.existsDist(dist)) throw new Error("Distribution not found!");
+        doc.dist = dist;
+      }
+      if (suite) {
+        if (!await partialConfig.existsSuite(dist, suite)) throw new Error("Suite/Component not found!");
+        doc.suite = suite;
+      }
+      if (Package) doc["control.Package"] = Package;
+      if (Arch) doc["control.Architecture"] = Arch;
+      if (Version) doc["control.Version"] = Version;
+      const packageInfo = await collection.find(doc).toArray();
+      if (!packageInfo) throw new Error("Package not found!");
+      return packageInfo.map(fixPackage);
+    }
+  } else {
+    // Internal Object
+    let packagesArray: packageSave[] = [];
+
+    // Add package to array
+    partialConfig.addPackage = async function addPackage(repo) {
+      const existsPackage = packagesArray.find((x) => x.control.Package === repo.control.Package && x.control.Version === repo.control.Version && x.control.Architecture === repo.control.Architecture && x.dist === repo.dist && x.suite === repo.suite && x.repository === repo.repository);
+      if (existsPackage) throw new Error("Package already exists!");
+      packagesArray.push(repo);
+      console.log("Added '%s', version: %s, Arch: %s, in to %s/%s", repo.control.Package, repo.control.Version, repo.control.Architecture, repo.dist, repo.suite);
+    }
+
+    // Delete package
+    partialConfig.deletePackage = async function deletePackage(repo) {
+      const index = packagesArray.findIndex((x) => x.control.Package === repo.control.Package && x.control.Version === repo.control.Version && x.control.Architecture === repo.control.Architecture && x.dist === repo.dist && x.suite === repo.suite && x.repository === repo.repository);
+      if (index === -1) throw new Error("Package not found!");
+      const packageDelete = packagesArray.splice(index, 1).at(-1);
+      console.info("Deleted '%s', version: %s, Arch: %s, from %s/%s", packageDelete.control.Package, packageDelete.control.Version, packageDelete.control.Architecture, packageDelete.dist, packageDelete.suite);
+      return packageDelete;
+    }
+
+    // Exists
+    partialConfig.existsDist = async function existsDist(dist) {
+      return packagesArray.find(x => x.dist === dist) ? true : false;
+    }
+    partialConfig.existsSuite = async function existsSuite(dist, suite) {
+      if (await partialConfig.existsDist(dist)) return packagesArray.find(x => x.dist === dist && x.suite === suite) ? true : false;
+      return false;
+    }
+
+    // Packages
+    partialConfig.getPackages = async function getPackages(dist, suite, Package, Arch, Version) {
+      if (dist && !await partialConfig.existsDist(dist)) throw new Error("Distribution not found!");
+      if (suite && !await partialConfig.existsSuite(dist, suite)) throw new Error("Suite/Component not found!");
+      const packageInfo = packagesArray.filter(x => (!dist || x.dist === dist) && (!suite || x.suite === suite) && (!Package || x.control.Package === Package) && (!Arch || x.control.Architecture === Arch) && (!Version || x.control.Version === Version));
+      if (!packageInfo.length) throw new Error("Package not found!");
+      return packageInfo;
+    }
+  }
+
+  // Return functions
+  return partialConfig as packageManegerV2;
+}

src/repoConfig.ts

@@ -1,12 +1,8 @@
-import coreUtils, { DebianPackage, DockerRegistry, extendFs, extendsCrypto, httpRequest } from "@sirherobrine23/coreutils";
+import coreUtils, { DockerRegistry, extendFs, httpRequest } from "@sirherobrine23/coreutils";
-import { Compressor as lzmaCompressor } from "lzma-native";
+import { format } from "node:util";
-import { Readable, Writable } from "node:stream";
-import { debianControl } from "@sirherobrine23/coreutils/src/deb.js";
-import { createGzip } from "node:zlib";
 import yaml from "yaml";
 import path from "node:path";
 import fs from "node:fs/promises";
-import { format } from "node:util";
 
 export type apt_config = {
   origin?: string,
@@ -81,6 +77,16 @@ export type backendConfig = Partial<{
       private: string,
       public: string,
       passphrase?: string
+    },
+    mongodb?: {
+      uri: string,
+      db?: string,
+      collection?: string,
+      /** On connect to database drop collection to run in empty data */
+      dropCollention?: boolean
+    },
+    packagesOptions?: {
+      uniqueVersion?: boolean,
     }
   },
   repositories: {
@@ -100,7 +106,6 @@ export async function saveConfig(filePath: string, config: backendConfig) {
 }
 
 export async function getConfig(config: string) {
-  const fixedConfig: backendConfig = {};
   let configData: backendConfig, avaiableToDirname = true;
   if (config.startsWith("http")) {
     avaiableToDirname = false;
@@ -128,15 +133,24 @@ export async function getConfig(config: string) {
       }
     }
   } else {
-    if (!await coreUtils.extendFs.exists(config)) throw new Error("config File not exists");
+    if (!await coreUtils.extendFs.exists(config)) throw new Error("config File not exists, return "+JSON.stringify(config));
     configData = yaml.parse(await fs.readFile(config, "utf8"));
   }
-  fixedConfig["apt-config"] = {};
+  if (typeof configData !== "object") throw new Error("Invalid config file");
+
+  const fixedConfig: backendConfig = {
+    "apt-config": {
+      packagesOptions: {
+        uniqueVersion: configData["apt-config"]?.packagesOptions?.uniqueVersion ?? false
+      }
+    },
+    repositories: {}
+  };
   if (configData["apt-config"]) {
     const rootData = configData["apt-config"];
     fixedConfig["apt-config"].portListen = rootData.portListen ?? 3000;
-    fixedConfig["apt-config"].poolPath = rootData.poolPath ?? path.join(process.cwd(), "apt-stream");
     fixedConfig["apt-config"].saveFiles = rootData.saveFiles ?? false;
+    if (rootData.poolPath) fixedConfig["apt-config"].poolPath = rootData.poolPath;
     if (fixedConfig["apt-config"].poolPath && !await extendFs.exists(fixedConfig["apt-config"].poolPath)) await fs.mkdir(fixedConfig["apt-config"].poolPath, {recursive: true});
     if (rootData.codename) fixedConfig["apt-config"].codename = rootData.codename;
     if (rootData.origin) fixedConfig["apt-config"].origin = rootData.origin;
@@ -155,6 +169,15 @@ export async function getConfig(config: string) {
         passphrase
       };
     }
+    if (rootData.mongodb) {
+      if (!rootData.mongodb.uri) throw new Error("mongodb.uri not defined");
+      fixedConfig["apt-config"].mongodb = {
+        uri: rootData.mongodb.uri,
+        db: rootData.mongodb.db ?? "apt-stream",
+        collection: rootData.mongodb.collection ?? "packages",
+        dropCollention: Boolean(rootData.mongodb.dropCollention ?? false)
+      };
+    }
   }
   if (fixedConfig["apt-config"].pgpKey) {
     const pgpKey = fixedConfig["apt-config"].pgpKey;
@@ -272,184 +295,4 @@ export async function getConfig(config: string) {
     });
   });
   return fixedConfig;
-}
-
-export type packageData = {
-  control: debianControl,
-  getStream: () => Readable|Promise<Readable>,
-  repositoryConfig?: repository
-}
-
-type distObject = {
-  [distribuition: string]: {
-    [suite: string]: {
-      [arch: string]: packageData[]
-    }
-  }
-}
-
-export class distManegerPackages {
-  public distribuitions: distObject = {};
-  public addDistribuition(distribuition: string) {
-    if (!this.distribuitions[distribuition]) this.distribuitions[distribuition] = {};
-    return this.distribuitions[distribuition];
-  }
-  public addSuite(distribuition: string, suite: string) {
-    if (!this.distribuitions[distribuition][suite]) this.distribuitions[distribuition][suite] = {};
-    return this.distribuitions[distribuition][suite];
-  }
-  public addArch(distribuition: string, suite: string, arch: string) {
-    if (!this.distribuitions[distribuition][suite][arch]) this.distribuitions[distribuition][suite][arch] = [];
-    return this.distribuitions[distribuition][suite][arch];
-  }
-
-  /**
-   * Register package in distribuition and suite
-   *
-   * @param distribuition
-   * @param suite
-   * @param arch
-   * @param control
-   * @param getStream
-   * @returns
-   */
-  public addPackage(distribuition: string, suite: string, packageData: packageData) {
-    this.addDistribuition(distribuition);
-    this.addSuite(distribuition, suite);
-    this.addArch(distribuition, suite, packageData.control.Architecture);
-    const currentPackages = this.distribuitions[distribuition][suite][packageData.control.Architecture];
-    if (currentPackages.some(pkg => pkg.control.Package === packageData.control.Package)) {
-      if (currentPackages.some(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package)) {
-        const index = currentPackages.findIndex(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package);
-        console.info("[INFO]: Replace %s, with version %s, target arch %s, index number %f", packageData.control.Package, packageData.control.Version, packageData.control.Architecture, index);
-        return this.distribuitions[distribuition][suite][packageData.control.Architecture][index] = packageData;
-      }
-    }
-    console.info("[INFO]: Add %s, with version %s, target arch %s", packageData.control.Package, packageData.control.Version, packageData.control.Architecture);
-    this.distribuitions[distribuition][suite][packageData.control.Architecture].push(packageData);
-    return packageData;
-  }
-
-  public deletePackage(distribuition: string, suite: string, arch: string, packageName: string, version: string) {
-    if (!this.distribuitions[distribuition]) throw new Error("Distribuition not exists");
-    if (!this.distribuitions[distribuition][suite]) throw new Error("Suite not exists");
-    if (!this.distribuitions[distribuition][suite][arch]) throw new Error("Arch not exists");
-    const index = this.distribuitions[distribuition][suite][arch].findIndex(pkg => pkg.control.Package === packageName && pkg.control.Version === version);
-    if (index === -1) throw new Error("Package not exists");
-    const data = this.distribuitions[distribuition][suite][arch][index];
-    this.distribuitions[distribuition][suite][arch].splice(index, 1);
-    return data;
-  }
-
-  public getDistribuition(distName: string) {
-    const dist = this.distribuitions[distName];
-    if (!dist) throw new Error("Distribuition not exists");
-    const suites = Object.keys(dist);
-    const suiteData = suites.map(suite => {
-      const Packages = Object.keys(dist[suite]).map(arch => dist[suite][arch].map(packageInfo => packageInfo.control)).flat();
-      return {
-        Suite: suite,
-        Archs: Object.keys(dist[suite]),
-        Packages
-      };
-    });
-
-    return {
-      dist: distName,
-      suites,
-      archs: [...(new Set(suiteData.map(suite => suite.Archs).flat()))],
-      suiteData,
-    };
-  }
-
-  public getAllDistribuitions() {
-    return Object.keys(this.distribuitions).map(dist => this.getDistribuition(dist)).flat();
-  }
-
-  public getPackageInfo(info: {dist: string, suite?: string, arch?: string, packageName?: string, version?: string}) {
-    const packageDateObject: {[k: string]: {[l: string]: {[a: string]: DebianPackage.debianControl[]}}} = {};
-    for (const dist in this.distribuitions) {
-      if (info.dist && info.dist !== dist) continue;
-      packageDateObject[dist] = {};
-      for (const suite in this.distribuitions[dist]) {
-        if (info.suite && info.suite !== suite) continue;
-        packageDateObject[dist][suite] = {};
-        for (const arch in this.distribuitions[dist][suite]) {
-          if (info.arch && info.arch !== arch) continue;
-          packageDateObject[dist][suite][arch] = this.distribuitions[dist][suite][arch].map(pkg => pkg.control).filter(pkg => (!info.packageName || pkg.Package === info.packageName) && (!info.version || pkg.Version === info.version));
-        }
-      }
-    }
-
-    if (info.dist) {
-      const dist = packageDateObject[info.dist];
-      if (info.suite) {
-        const suite = dist[info.suite];
-        if (info.arch) {
-          const arch = suite[info.arch];
-          if (info.packageName) return arch.find(pkg => pkg.Package === info.packageName && (!info.version || pkg.Version === info.version));
-          return arch;
-        }
-      }
-      return dist;
-    }
-    return packageDateObject;
-  }
-
-  public async getPackageStream(distribuition: string, suite: string, arch: string, packageName: string, version: string) {
-    if (!this.distribuitions[distribuition]) throw new Error("Distribuition not exists");
-    if (!this.distribuitions[distribuition][suite]) throw new Error("Suite not exists");
-    if (!this.distribuitions[distribuition][suite][arch]) throw new Error("Arch not exists");
-    const packageData = this.distribuitions[distribuition][suite][arch].find(pkg => pkg.control.Package === packageName && pkg.control.Version === version);
-    if (!packageData) throw new Error("Package not exists");
-    return Promise.resolve(packageData.getStream()).then(stream => ({control: packageData.control, repository: packageData.repositoryConfig, stream}));
-  }
-
-  public async createPackages(options?: {compress?: "gzip" | "xz", writeStream?: Writable, singlePackages?: boolean, dist?: string, package?: string, arch?: string, suite?: string}) {
-    const distribuition = this.distribuitions;
-    const rawWrite = new Readable({read(){}});
-    let size = 0, addbreak = false, hash: ReturnType<typeof extendsCrypto.createHashAsync>|undefined;
-    if (options?.compress === "gzip") {
-      const gzip = rawWrite.pipe(createGzip({level: 9}));
-      if (options?.writeStream) gzip.pipe(options.writeStream);
-      hash = extendsCrypto.createHashAsync("all", gzip);
-      gzip.on("data", (chunk) => size += chunk.length);
-    } else if (options?.compress === "xz") {
-      const lzma = rawWrite.pipe(lzmaCompressor());
-      if (options?.writeStream) lzma.pipe(options.writeStream);
-      hash = extendsCrypto.createHashAsync("all", lzma);
-      lzma.on("data", (chunk) => size += chunk.length);
-    } else {
-      if (options?.writeStream) rawWrite.pipe(options.writeStream);
-      hash = extendsCrypto.createHashAsync("all", rawWrite);
-      rawWrite.on("data", (chunk) => size += chunk.length);
-    }
-
-    for (const dist in distribuition) {
-      if (options?.dist && options.dist !== dist) continue;
-      const suites = distribuition[dist];
-      for (const suite in suites) {
-        if (options?.suite && options.suite !== suite) continue;
-        const archs = suites[suite];
-        for (const arch in archs) {
-          if (arch !== "all" && (options?.arch && options.arch !== arch)) continue;
-          const packages = archs[arch];
-          for (const {control} of packages) {
-            if (!control.Size) continue;
-            if (!(control.SHA1 || control.SHA256 || control.MD5sum)) continue;
-            if (options?.package && options.package !== control.Package) continue;
-            if (addbreak) rawWrite.push("\n\n"); else addbreak = true;
-            control["Filename"] = poolLocationPackage(dist, suite, arch, control.Package, control.Version);
-            const Data = Object.keys(control).map(key => `${key}: ${control[key]}`);
-            rawWrite.push(Data.join("\n"));
-            if (options?.singlePackages) break;
-          }
-        }
-      }
-    }
-
-    rawWrite.push(null);
-    if (hash) return hash.then(hash => ({...hash, size}));
-    return null;
-  }
 }

tsconfig.json

@@ -21,6 +21,8 @@
     "src/**/*.test.ts"
   ],
   "ts-node": {
-    "esm": true
+    "esm": true,
+    "experimentalResolver": true,
+    "experimentalSpecifierResolution": "node"
   }
 }