package-lock.json

@@ -14,6 +14,7 @@
         "lzma-native": "^8.0.6",
         "mongodb": "^4.13.0",
         "openpgp": "^5.5.0",
+        "semver": "^7.3.8",
         "tar": "^6.1.13",
         "yaml": "^2.2.1",
         "yargs": "^17.6.2"
@@ -25,6 +26,7 @@
         "@types/express": "^4.17.15",
         "@types/lzma-native": "^4.0.1",
         "@types/node": "^18.11.18",
+        "@types/semver": "^7.3.13",
         "@types/tar": "^6.1.3",
         "@types/yargs": "^17.0.19",
         "ts-node": "^10.9.1",
@@ -1414,6 +1416,12 @@
       "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
       "dev": true
     },
+    "node_modules/@types/semver": {
+      "version": "7.3.13",
+      "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz",
+      "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
+      "dev": true
+    },
     "node_modules/@types/serve-static": {
       "version": "1.15.0",
       "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz",
@@ -3204,6 +3212,20 @@
         "node": ">=v12.22.7"
       }
     },
+    "node_modules/semver": {
+      "version": "7.3.8",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
+      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/send": {
       "version": "0.18.0",
       "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
@@ -4966,6 +4988,12 @@
       "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
       "dev": true
     },
+    "@types/semver": {
+      "version": "7.3.13",
+      "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz",
+      "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
+      "dev": true
+    },
     "@types/serve-static": {
       "version": "1.15.0",
       "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz",
@@ -6280,6 +6308,14 @@
         "xmlchars": "^2.2.0"
       }
     },
+    "semver": {
+      "version": "7.3.8",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
+      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
+      "requires": {
+        "lru-cache": "^6.0.0"
+      }
+    },
     "send": {
       "version": "0.18.0",
       "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",

package.json

@@ -38,6 +38,7 @@
     "@types/express": "^4.17.15",
     "@types/lzma-native": "^4.0.1",
     "@types/node": "^18.11.18",
+    "@types/semver": "^7.3.13",
     "@types/tar": "^6.1.3",
     "@types/yargs": "^17.0.19",
     "ts-node": "^10.9.1",
@@ -49,6 +50,7 @@
     "lzma-native": "^8.0.6",
     "mongodb": "^4.13.0",
     "openpgp": "^5.5.0",
+    "semver": "^7.3.8",
     "tar": "^6.1.13",
     "yaml": "^2.2.1",
     "yargs": "^17.6.2"

src/express_route.ts

@@ -2,11 +2,14 @@ import { Compressor as lzmaCompress } from "lzma-native";
 import { createGzip } from "node:zlib";
 import { getConfig } from "./repoConfig.js";
 import { Readable } from "node:stream";
-import package_maneger from "./packagesData.js";
+import package_maneger, { packageSave } from "./packagesData.js";
 import coreUtils from "@sirherobrine23/coreutils";
 import cluster from "node:cluster";
 import express from "express";
 import openpgp from "openpgp";
+import semver from "semver";
+import path from "node:path";
+import os from "node:os";
 
 export default async function initApp(config: string) {
   const packageConfig = await getConfig(config);
@@ -22,6 +25,18 @@ export default async function initApp(config: string) {
     res.on("close", () => console.log("[%s, cluserID: %s]: Path: %s, Method: %s, IP: %s, Status: %f", new Date().toISOString(), cluserID, req.path, req.method, req.ip, res.statusCode));
     next();
   });
+
+  // Info
+  app.get("/", ({res}) => {
+    return res.json({
+      cluster: cluster.worker?.id ?? "No clustered",
+      cpuCores: os.cpus().length || "Unknown CPU core",
+      hostArch: process.arch,
+      hostPlatform: process.platform,
+      nodeVersion: process.version,
+    });
+  });
+
   app.get("/pool/:dist/:suite/:package/:arch/:version/download.deb", async ({params: {dist, suite, package: packageName, arch, version}}, res, next) => {
     try {
       const data = (await packageManeger.getPackages(dist, suite, packageName, arch, version))?.at(-1);
@@ -48,23 +63,59 @@ export default async function initApp(config: string) {
     return old;
   }, {}))).catch(next));
 
-  app.get(["/", "/pool"], ({}, res, next) => packageManeger.getPackages().then(data => res.json(data.reduce((old, current) => {
+  app.get("/pool", ({}, res, next) => packageManeger.getPackages().then(data => res.json(data.reduce((old, current) => {
     if (!old[current.dist]) old[current.dist] = {};
     if (!old[current.dist][current.suite]) old[current.dist][current.suite] = [];
     old[current.dist][current.suite].push(current.control);
     return old;
   }, {}))).catch(next));
 
+  // Package list
+  app.get("/source(s)?(.list)?", async (req, res, next) => {
+    try {
+      const dists = await Promise.all(await packageManeger.getDists().then(data => data.map(dist => packageManeger.getDistInfo(dist).then(data => ({dist, data})))));
+      res.set("Content-Type", "text/plain");
+      const remotePath = path.posix.resolve(req.baseUrl + req.path, ".."), protocol = req.headers["x-forwarded-proto"] ?? req.protocol, hostname = process.env["RAILWAY_STATIC_URL"] ?? `${req.hostname}:${req.socket.localPort}`, host = packageConfig["apt-config"]?.sourcesHost ?? `${protocol}://${hostname}${remotePath}`;
+      const data = [];
+      for (const {dist, data: {suites}} of dists) data.push(`deb ${host} ${dist} ${suites.join(" ")}`);
+      return res.send(data.join("\n"));
+    } catch (err) {return next(err);}
+  });
+
   // Dists info
   app.get("/dists", ({}, res, next) => packageManeger.getDists().then(data => res.json(data)).catch(next));
   app.get("/dists/:dist", ({params: {dist}}, res, next) => packageManeger.getDistInfo(dist).then(data => res.json(data)).catch(next));
+  app.get("/dists/:dist/source(s)?(.list)?", async (req, res, next) => {
+    const {dist} = req.params;
+    try {
+      const data = await packageManeger.getDistInfo(dist);
+      res.set("Content-Type", "text/plain");
+      const remotePath = path.posix.resolve(req.baseUrl + req.path, ".."), protocol = req.headers["x-forwarded-proto"] ?? req.protocol, hostname = process.env["RAILWAY_STATIC_URL"] ?? `${req.hostname}:${req.socket.localPort}`, host = packageConfig["apt-config"]?.sourcesHost ?? `${protocol}://${hostname}${remotePath}`;
+      return res.send(`deb ${host} ${dist} ${data.suites.join(" ")}`);
+    } catch (err) {return next(err);}
+  });
 
   // Create Package, Package.gz and Package.xz
-  async function createPackages(dist: string, suite: string, arch: string) {
+  function createPackages(packages: packageSave[], dist: string, suite: string) {
-    if (!await packageManeger.existsDist(dist)) throw new Error("Distribution not exists");
-    if (!await packageManeger.existsSuite(dist, suite)) throw new Error("Suite not exists");
-    const packages = (await packageManeger.getPackages(dist, suite, undefined, arch)).concat(arch !== "all" ? await packageManeger.getPackages(dist, suite, undefined, "all") : []);
     if (!packages.length) throw new Error("Check is dist or suite have packages");
+    const reduced = packages.reduce((old, current) => {
+      if (!old[current.control.Package]) old[current.control.Package] = [];
+      old[current.control.Package].push(current);
+      return old;
+    }, {} as {[packageName: string]: packageSave[]});
+    packages = [];
+    Object.keys(reduced).forEach(packageName => {
+      reduced[packageName] = reduced[packageName].sort((b, a) => {
+        const aVersion = semver.valid(semver.coerce(a.control.Version) ?? ""), bVersion = semver.valid(semver.coerce(b.control.Version) ?? "");
+        return semver.compare(aVersion, bVersion);
+      });
+      if (!packageConfig["apt-config"]?.packagesOptions?.uniqueVersion) packages.push(...reduced[packageName]);
+      else {
+        const at = reduced[packageName].at(0);
+        if (at) packages.push(at);
+      }
+    });
+
     let rawSize = 0, gzipSize = 0, lzmaSize = 0;
     const mainReadstream = new Readable({read(){}}), rawSUMs = coreUtils.extendsCrypto.createHashAsync("all", mainReadstream).then(hash => ({size: rawSize, hash}));
     const gzip = mainReadstream.pipe(createGzip()), gzipSUMs = coreUtils.extendsCrypto.createHashAsync("all", gzip).then(hash => ({size: gzipSize, hash}));
@@ -72,7 +123,6 @@ export default async function initApp(config: string) {
     mainReadstream.on("data", data => rawSize += data.length);
     gzip.on("data", data => gzipSize += data.length);
     lzma.on("data", data => lzmaSize += data.length);
-
     let fist = true;
     for (const {control} of packages) {
       if (!(control.Size && (control.MD5sum || control.SHA256 || control.SHA1))) continue;
@@ -94,12 +144,14 @@ export default async function initApp(config: string) {
     };
   }
 
-  app.get("/dists/(./)?:dist/:suite/binary-:arch/Packages(.(xz|gz)|)", async ({params: {dist, suite, arch}, path: reqPath}, res, next) => createPackages(dist, suite, arch).then(packages => {
+  app.get("/dists/(./)?:dist/:suite/binary-:arch/Packages(.(xz|gz)|)", async ({params: {dist, suite, arch}, path: reqPath}, res, next) => {
-    if (reqPath.endsWith(".gz")) return packages.gzip.pipe(res);
+    try {
-    else if (reqPath.endsWith(".xz")) return packages.lzma.pipe(res);
+      const packages = createPackages(await packageManeger.getPackages(dist, suite, undefined, arch), dist, suite);
-    else return packages.raw.pipe(res);
+      if (reqPath.endsWith(".gz")) return packages.gzip.pipe(res);
-  }).catch(next));
+      else if (reqPath.endsWith(".xz")) return packages.lzma.pipe(res);
-
+      else return packages.raw.pipe(res);
+    } catch (err) {return next(err);}
+  });
   // Release
   async function createRelease(dist: string) {
     if (!await packageManeger.existsDist(dist)) throw new Error("Dist exists");
@@ -140,7 +192,8 @@ export default async function initApp(config: string) {
       Release.SHA1 = [];
       Release.MD5sum = [];
       const files = await Promise.all(Archs.map(async Arch => Promise.all(Components.map(async Component => {
-        const {SUMs} = await createPackages(dist, Component, Arch);
+        const archPackages = packagesArray.filter(x => x.control.Architecture === Arch && x.suite === Component);
+        const {SUMs} = createPackages(archPackages, dist, Component);
         return [
           {
             file: `${Component}/binary-${Arch}/Packages`,

src/packagesData.ts

@@ -4,10 +4,10 @@ import { MongoClient, ServerApiVersion, Filter } from "mongodb";
 import { apt_config, backendConfig, repository } from "./repoConfig.js";
 import { getPackages as mirror } from "./mirror.js";
 import { Readable } from "node:stream";
+import { format } from "node:util";
 import cluster from "node:cluster";
 import path from "node:path";
 import tar from "tar";
-import { format } from "node:util";
 
 export type packageSave = {
   dist: string,
@@ -365,15 +365,12 @@ export default async function packageManeger(config: backendConfig): Promise<pac
     function fixPackage(data: packageSave): packageSave {
       if (!data.restoreFileStream) throw new Error("cannot restore file stream!");
       data.getFileStream = async function getFileStream() {
-        if (data.restoreFileStream.from === "github_release"|| data.restoreFileStream.from === "github_tree") return coreUtils.httpRequest.pipeFetch(data.restoreFileStream.url);
+        if (data.restoreFileStream.fileUrl) return coreUtils.httpRequest.pipeFetch(data.restoreFileStream.fileUrl);
-        else if (data.restoreFileStream.from === "google_drive" && data.repository.from === "google_drive") {
+        if (data.restoreFileStream.from === "google_drive" && data.repository.from === "google_drive") {
-          const googleDriver = await coreUtils.googleDriver.GoogleDriver(data.repository.appSettings.client_id, data.repository.appSettings.client_secret, {token: data.repository.appSettings.token});
+          const { appSettings } = data.repository;
-          return googleDriver.getFileStream(data.restoreFileStream.fileId);
+          const googleDrive = await coreUtils.googleDriver.GoogleDriver(appSettings.client_id, appSettings.client_secret, {token: appSettings.token});
-        } else if (data.restoreFileStream.from === "oracle_bucket" && data.repository.from === "oracle_bucket") {
+          return googleDrive.getFileStream(data.restoreFileStream.fileId);
-          const oracleBucket = await coreUtils.oracleBucket(data.repository.region as any, data.repository.bucketName, data.repository.bucketNamespace, data.repository.auth);
+        } else if (data.restoreFileStream.from === "oci" && data.repository.from === "oci") {
-          return oracleBucket.getFileStream(data.restoreFileStream.fileName);
-        } else if (data.restoreFileStream.from === "mirror" && data.repository.from === "mirror") return coreUtils.httpRequest.pipeFetch(data.restoreFileStream.fileUrl);
-        else if (data.restoreFileStream.from === "oci" && data.repository.from === "oci") {
           const oci = await coreUtils.DockerRegistry(data.repository.image);
           return new Promise((done, reject) => {
             oci.blobLayerStream(data.restoreFileStream.digest).then((stream) => {

src/repoConfig.ts

@@ -84,6 +84,9 @@ export type backendConfig = Partial<{
       collection?: string,
       /** On connect to database drop collection to run in empty data */
       dropCollention?: boolean
+    },
+    packagesOptions?: {
+      uniqueVersion?: boolean,
     }
   },
   repositories: {
@@ -136,7 +139,11 @@ export async function getConfig(config: string) {
   if (typeof configData !== "object") throw new Error("Invalid config file");
 
   const fixedConfig: backendConfig = {
-    "apt-config": {},
+    "apt-config": {
+      packagesOptions: {
+        uniqueVersion: configData["apt-config"]?.packagesOptions?.uniqueVersion ?? false
+      }
+    },
     repositories: {}
   };
   if (configData["apt-config"]) {