package.json

@@ -48,6 +48,7 @@
     "cron": "^2.1.0",
     "express": "^4.18.2",
     "lzma-native": "^8.0.6",
+    "mongodb": "^4.13.0",
     "openpgp": "^5.5.0",
     "tar": "^6.1.13",
     "yaml": "^2.2.1",

src/express_route.ts

@@ -12,8 +12,8 @@ import path from "node:path";
 
 export default async function main(configPath: string) {
   // Load config
-  const packInfos = new distManegerPackages()
   let repositoryConfig = await getConfig(configPath);
+  const packInfos = new distManegerPackages(repositoryConfig);
 
   // Express app
   const app = express();
@@ -34,12 +34,12 @@ export default async function main(configPath: string) {
   });
 
   // Sources list
-  app.get(["/source_list", "/sources.list"], (req, res) => {
+  app.get(["/source_list", "/sources.list"], async (req, res) => {
     const remotePath = path.posix.resolve(req.baseUrl + req.path, ".."),
       protocol = req.headers["x-forwarded-proto"] ?? req.protocol,
       hostname = process.env["RAILWAY_STATIC_URL"] ?? `${req.hostname}:${req.socket.localPort}`,
       host = repositoryConfig["apt-config"]?.sourcesHost ?? `${protocol}://${hostname}${remotePath}`,
-      concatPackage = packInfos.getAllDistribuitions(),
+      concatPackage = await packInfos.getAllDistribuitions(),
       type = req.query.type ?? req.query.t,
       Conflicting = !!(req.query.conflicting ?? req.query.c);
     if (type === "json") {
@@ -54,12 +54,12 @@ export default async function main(configPath: string) {
   });
 
   // Download
-  app.get(["/pool", "/"], (_req, res) => res.json(packInfos.getAllDistribuitions()));
+  app.get(["/pool", "/"], async (_req, res) => res.json(await packInfos.getAllDistribuitions()));
-  app.get("/pool/:dist", (req, res) => res.json(packInfos.getDistribuition(req.params.dist)));
+  app.get("/pool/:dist", async (req, res) => res.json(await packInfos.getDistribuition(req.params.dist)));
-  app.get("/pool/:dist/:suite", ({params: {dist, suite}}, res) => res.json(packInfos.getPackageInfo({dist, suite})));
+  app.get("/pool/:dist/:suite", async ({params: {dist, suite}}, res) => res.json(await packInfos.getPackageInfo({dist, suite})));
-  app.get("/pool/:dist/:suite/:arch", ({params: {dist, suite, arch}}, res) => res.json(packInfos.getPackageInfo({dist, suite, arch})));
+  app.get("/pool/:dist/:suite/:arch", async ({params: {dist, suite, arch}}, res) => res.json(await packInfos.getPackageInfo({dist, suite, arch})));
-  app.get("/pool/:dist/:suite/:arch/:packageName", ({params: {dist, suite, arch, packageName}}, res) => res.json(packInfos.getPackageInfo({dist, suite, arch, packageName})));
+  app.get("/pool/:dist/:suite/:arch/:packageName", async ({params: {dist, suite, arch, packageName}}, res) => res.json(await packInfos.getPackageInfo({dist, suite, arch, packageName})));
-  app.get("/pool/:dist/:suite/:arch/:packageName/:version", ({params: {dist, suite, arch, packageName, version}}, res) => res.json(packInfos.getPackageInfo({dist, suite, arch, packageName, version})));
+  app.get("/pool/:dist/:suite/:arch/:packageName/:version", async ({params: {dist, suite, arch, packageName, version}}, res) => res.json(await packInfos.getPackageInfo({dist, suite, arch, packageName, version})));
   app.get("/pool/:dist/:suite/:arch/:packageName/:version/download.deb", async ({params: {dist, suite, arch, packageName, version}}, res, next) => packInfos.getPackageStream(dist, suite, arch, packageName, version).then(data => data.stream.pipe(res.writeHead(200, {"Content-Type": "application/x-debian-package", "Content-Length": data.control.Size, "Content-Disposition": `attachment; filename="${packageName}_${version}_${arch}.deb"`, "SHA256_hash": data.control.SHA256, "MD5Sum_hash": data.control.MD5sum}))).catch(next));
 
   app.get("/dists/(./)?:dist/:suite/binary-:arch/Packages(.(xz|gz)|)", (req, res) => {
@@ -99,7 +99,7 @@ export default async function main(configPath: string) {
 
   // Release
   async function createReleaseV1(dist: string) {
-    const { suites, archs } = packInfos.getDistribuition(dist);
+    const { suites, archs } = await packInfos.getDistribuition(dist);
     const distConfig = repositoryConfig.repositories[dist];
     if (!distConfig) throw new Error("Dist not found");
     const ReleaseLines = [];

src/mirror.ts

@@ -82,7 +82,6 @@ export async function getPackages(uri: string, options: {dist: string, suite?: s
   const hashs: {file: string, Package: DebianPackage.debianControl}[] = [];
   async function addPackages(dist: string, file: string, fn?: (data: DebianPackage.debianControl) => void) {
     const urlRequest = `${uri}/dists/${dist}/${file}`;
-    console.log(`Requesting ${urlRequest}`);
     await new Promise<void>(async (done, reject) => {
       const stream = (urlRequest.endsWith(".gz")||urlRequest.endsWith(".xz")) ? (await httpRequest.pipeFetch(urlRequest)).pipe(urlRequest.endsWith(".gz") ? zlib.createGunzip() : lzmaDecompressor()) : await httpRequest.pipeFetch(urlRequest);
       stream.on("error", (err: any) => {

src/repoConfig.ts

@@ -1,12 +1,15 @@
 import coreUtils, { DebianPackage, DockerRegistry, extendFs, extendsCrypto, httpRequest } from "@sirherobrine23/coreutils";
+import { createReadStream, createWriteStream } from "node:fs";
+import { MongoClient, ServerApiVersion } from "mongodb";
 import { Compressor as lzmaCompressor } from "lzma-native";
 import { Readable, Writable } from "node:stream";
 import { debianControl } from "@sirherobrine23/coreutils/src/deb.js";
 import { createGzip } from "node:zlib";
+import { format } from "node:util";
 import yaml from "yaml";
 import path from "node:path";
 import fs from "node:fs/promises";
-import { format } from "node:util";
+import tar from "tar";
 
 export type apt_config = {
   origin?: string,
@@ -81,6 +84,11 @@ export type backendConfig = Partial<{
       private: string,
       public: string,
       passphrase?: string
+    },
+    mongodb?: {
+      uri: string,
+      db?: string,
+      collection?: string,
     }
   },
   repositories: {
@@ -155,6 +163,14 @@ export async function getConfig(config: string) {
         passphrase
       };
     }
+    if (rootData.mongodb) {
+      if (!rootData.mongodb.uri) throw new Error("mongodb.uri not defined");
+      fixedConfig["apt-config"].mongodb = {
+        uri: rootData.mongodb.uri,
+        db: rootData.mongodb.db ?? "apt-stream",
+        collection: rootData.mongodb.collection ?? "packages"
+      };
+    }
   }
   if (fixedConfig["apt-config"].pgpKey) {
     const pgpKey = fixedConfig["apt-config"].pgpKey;
@@ -288,17 +304,119 @@ type distObject = {
   }
 }
 
+type dbDist = {
+  control: debianControl,
+  repositoryConfig?: repository,
+  dist: string,
+  suite: string,
+  getfile: repository & {file: string, blobLayer?: string}
+};
+
 export class distManegerPackages {
   public distribuitions: distObject = {};
-  public addDistribuition(distribuition: string) {
+  public mongoClinte?: MongoClient;
+  public internalDist: distObject = {};
+  public config: backendConfig;
+  constructor(config: backendConfig) {
+    this.config = config;
+    if (config["apt-config"]?.mongodb) {
+      this.mongoClinte = new MongoClient(config["apt-config"]?.mongodb?.uri, {
+        serverApi: ServerApiVersion.v1,
+      });
+      this.mongoClinte.connect();
+    }
+  }
+
+  async getPackages(dist?: string, suite?: string): Promise<distObject> {
+    let repo: distObject = {};
+    if (!this.mongoClinte) repo = this.distribuitions;
+    else {
+      if (dist && typeof dist !== "string") throw new Error("dist must be a string");
+      if (suite && typeof suite !== "string") throw new Error("suite must be a string");
+      const saveFile = this.config["apt-config"]?.saveFiles;
+      const rootPool = this.config["apt-config"]?.poolPath;
+      const collection = this.mongoClinte.db(this.config["apt-config"]?.mongodb?.db ?? "packages").collection<dbDist>(this.config["apt-config"]?.mongodb.collection ?? "packages");
+      for (const dataDB of await collection.find({dist, suite}).toArray()) {
+        const repository = dataDB.getfile, control = dataDB.control;
+        if (dist && dist !== dataDB.dist) continue;
+        if (suite && suite !== dataDB.suite) continue;
+        async function getStream() {
+          if (repository.from === "mirror") {
+            const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+            if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+            return coreUtils.httpRequest.pipeFetch(dataDB.getfile.file);
+          } else if (repository.from === "oci") {
+            const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+            if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+            const registry = await coreUtils.DockerRegistry(repository.image, repository.platfom_target);
+            return new Promise<Readable>((done, reject) => registry.blobLayerStream(dataDB.getfile.blobLayer).then(stream => {
+              stream.on("error", reject);
+              stream.pipe(tar.list({
+                async onentry(getEntry) {
+                  if (getEntry.path !== dataDB.getfile.file) return null;
+                  if (saveFile) {
+                    const mainPath = path.resolve(filePool, "..");
+                    if (!await extendFs.exists(mainPath)) await fs.mkdir(mainPath, {recursive: true});
+                    getEntry.pipe(createWriteStream(filePool));
+                  }
+                  return done(getEntry as any);
+                }
+              // @ts-ignore
+              }).on("error", reject));
+            }).catch(reject));
+          } else if (repository.from === "github_release" || repository.from === "github_tree") {
+            const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+            if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+            return coreUtils.httpRequest.pipeFetch({
+              url: dataDB.getfile.file,
+              headers: (repository.token ? {"Authorization": `token ${repository.token}`} : {})
+            })
+          } else if (repository.from === "google_drive") {
+            const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+            if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+            const client_id = repository.appSettings.client_id;
+            const client_secret = repository.appSettings.client_secret;
+            const token = repository.appSettings.token;
+            const googleDriver = await coreUtils.googleDriver.GoogleDriver(client_id, client_secret, {
+              token,
+              async authCallback(url, token) {
+                if (url) console.log("Please visit this url to auth google driver: %s", url);
+                else console.log("Google driver auth success, please save token to config file, token: %s", token);
+              },
+            });
+            return googleDriver.getFileStream(dataDB.getfile.file);
+          } else if (repository.from === "oracle_bucket") {
+            const filePool = path.join(rootPool, control.Package.slice(0, 1), `${control.Package}_${control.Architecture}_${control.Version}.deb`);
+            if (saveFile && await extendFs.exists(filePool)) return createReadStream(filePool);
+            const oracleBucket = await coreUtils.oracleBucket(repository.region as any, repository.bucketName, repository.bucketNamespace, repository.auth);
+            return oracleBucket.getFileStream(dataDB.getfile.file);
+          }
+
+          return null;
+        }
+        if (!repo[dataDB.dist]) repo[dataDB.dist] = {};
+        if (!repo[dataDB.dist][dataDB.suite]) repo[dataDB.dist][dataDB.suite] = {};
+        if (!repo[dataDB.dist][dataDB.suite][control.Architecture]) repo[dataDB.dist][dataDB.suite][control.Architecture] = [];
+        repo[dataDB.dist][dataDB.suite][control.Architecture].push({
+          getStream,
+          repositoryConfig: dataDB.repositoryConfig,
+          control,
+        });
+        console.log("add package %s %s %s %s", dataDB.dist, dataDB.suite, control.Architecture, control.Package);
+      };
+    }
+    return repo;
+  }
+
+  public async addDistribuition(distribuition: string) {
     if (!this.distribuitions[distribuition]) this.distribuitions[distribuition] = {};
     return this.distribuitions[distribuition];
   }
-  public addSuite(distribuition: string, suite: string) {
+  public async addSuite(distribuition: string, suite: string) {
     if (!this.distribuitions[distribuition][suite]) this.distribuitions[distribuition][suite] = {};
     return this.distribuitions[distribuition][suite];
   }
-  public addArch(distribuition: string, suite: string, arch: string) {
+  public async addArch(distribuition: string, suite: string, arch: string) {
     if (!this.distribuitions[distribuition][suite][arch]) this.distribuitions[distribuition][suite][arch] = [];
     return this.distribuitions[distribuition][suite][arch];
   }
@@ -313,20 +431,39 @@ export class distManegerPackages {
    * @param getStream
    * @returns
    */
-  public addPackage(distribuition: string, suite: string, packageData: packageData) {
+  public async addPackage(distribuition: string, suite: string, packageData: packageData) {
+    const dist = await this.getPackages(distribuition, suite);
     this.addDistribuition(distribuition);
     this.addSuite(distribuition, suite);
     this.addArch(distribuition, suite, packageData.control.Architecture);
-    const currentPackages = this.distribuitions[distribuition][suite][packageData.control.Architecture];
+    const currentPackages = dist[distribuition]?.[suite]?.[packageData.control.Architecture] ?? [];
-    if (currentPackages.some(pkg => pkg.control.Package === packageData.control.Package)) {
+    if (!this.mongoClinte) {
-      if (currentPackages.some(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package)) {
+      if (currentPackages.some(pkg => pkg.control.Package === packageData.control.Package)) {
-        const index = currentPackages.findIndex(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package);
+        if (currentPackages.some(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package)) {
-        console.info("[INFO]: Replace %s, with version %s, target arch %s, index number %f", packageData.control.Package, packageData.control.Version, packageData.control.Architecture, index);
+          const index = currentPackages.findIndex(pkg => pkg.control.Version === packageData.control.Version && pkg.control.Package === packageData.control.Package);
-        return this.distribuitions[distribuition][suite][packageData.control.Architecture][index] = packageData;
+          return dist[distribuition][suite][packageData.control.Architecture][index] = packageData;
+        }
       }
+      dist[distribuition][suite][packageData.control.Architecture].push(packageData);
+    } else {
+      const collection = this.mongoClinte.db(this.config["apt-config"]?.mongodb?.db ?? "packages").collection(this.config["apt-config"]?.mongodb.collection ?? "packages");
+      const currentPackages = await collection.findOne({
+        dist: distribuition,
+        suite,
+        cotrol: {
+          Package: packageData.control.Package,
+          Version: packageData.control.Version,
+          Architecture: packageData.control.Architecture,
+        }
+      });
+      if (currentPackages) await collection.deleteOne(currentPackages);
+      await collection.insertOne({
+        dist: distribuition,
+        suite,
+        control: packageData.control,
+        repository: packageData.repositoryConfig
+      });
     }
-    console.info("[INFO]: Add %s, with version %s, target arch %s", packageData.control.Package, packageData.control.Version, packageData.control.Architecture);
-    this.distribuitions[distribuition][suite][packageData.control.Architecture].push(packageData);
     return packageData;
   }
 
@@ -341,9 +478,9 @@ export class distManegerPackages {
     return data;
   }
 
-  public getDistribuition(distName: string) {
+  public async getDistribuition(distName: string) {
-    const dist = this.distribuitions[distName];
+    const dist = (await this.getPackages(distName))[distName];
-    if (!dist) throw new Error("Distribuition not exists");
+    if (!dist) return null;
     const suites = Object.keys(dist);
     const suiteData = suites.map(suite => {
       const Packages = Object.keys(dist[suite]).map(arch => dist[suite][arch].map(packageInfo => packageInfo.control)).flat();
@@ -362,21 +499,23 @@ export class distManegerPackages {
     };
   }
 
-  public getAllDistribuitions() {
+  public async getAllDistribuitions() {
-    return Object.keys(this.distribuitions).map(dist => this.getDistribuition(dist)).flat();
+    const dist = await this.getPackages();
+    return (await Promise.all(Object.keys(dist).map(dist => this.getDistribuition(dist)))).flat().filter(Boolean);
   }
 
-  public getPackageInfo(info: {dist: string, suite?: string, arch?: string, packageName?: string, version?: string}) {
+  public async getPackageInfo(info: {dist: string, suite?: string, arch?: string, packageName?: string, version?: string}) {
     const packageDateObject: {[k: string]: {[l: string]: {[a: string]: DebianPackage.debianControl[]}}} = {};
-    for (const dist in this.distribuitions) {
+    const distData = await this.getPackages(info.dist, info.suite);
+    for (const dist in distData) {
       if (info.dist && info.dist !== dist) continue;
       packageDateObject[dist] = {};
-      for (const suite in this.distribuitions[dist]) {
+      for (const suite in distData[dist]) {
         if (info.suite && info.suite !== suite) continue;
         packageDateObject[dist][suite] = {};
-        for (const arch in this.distribuitions[dist][suite]) {
+        for (const arch in distData[dist][suite]) {
           if (info.arch && info.arch !== arch) continue;
-          packageDateObject[dist][suite][arch] = this.distribuitions[dist][suite][arch].map(pkg => pkg.control).filter(pkg => (!info.packageName || pkg.Package === info.packageName) && (!info.version || pkg.Version === info.version));
+          packageDateObject[dist][suite][arch] = distData[dist][suite][arch].map(pkg => pkg.control).filter(pkg => (!info.packageName || pkg.Package === info.packageName) && (!info.version || pkg.Version === info.version));
         }
       }
     }
@@ -397,16 +536,17 @@ export class distManegerPackages {
   }
 
   public async getPackageStream(distribuition: string, suite: string, arch: string, packageName: string, version: string) {
-    if (!this.distribuitions[distribuition]) throw new Error("Distribuition not exists");
+    const dist = (await this.getPackages(distribuition))[distribuition];
-    if (!this.distribuitions[distribuition][suite]) throw new Error("Suite not exists");
+    if (!dist) throw new Error("Distribuition not exists");
-    if (!this.distribuitions[distribuition][suite][arch]) throw new Error("Arch not exists");
+    if (!dist[suite]) throw new Error("Suite not exists");
-    const packageData = this.distribuitions[distribuition][suite][arch].find(pkg => pkg.control.Package === packageName && pkg.control.Version === version);
+    if (!dist[suite][arch]) throw new Error("Arch not exists");
+    const packageData = dist[suite][arch].find(pkg => pkg.control.Package === packageName && pkg.control.Version === version);
     if (!packageData) throw new Error("Package not exists");
     return Promise.resolve(packageData.getStream()).then(stream => ({control: packageData.control, repository: packageData.repositoryConfig, stream}));
   }
 
   public async createPackages(options?: {compress?: "gzip" | "xz", writeStream?: Writable, singlePackages?: boolean, dist?: string, package?: string, arch?: string, suite?: string}) {
-    const distribuition = this.distribuitions;
+    const distribuition = await this.getPackages(options?.dist);
     const rawWrite = new Readable({read(){}});
     let size = 0, addbreak = false, hash: ReturnType<typeof extendsCrypto.createHashAsync>|undefined;
     if (options?.compress === "gzip") {