.github/workflows/nightly_build.yaml

@@ -28,9 +28,7 @@ jobs:
     - name: Build image
       uses: docker/build-push-action@v4
       with:
-        cache-from: ${{ github.event_name == 'push' && 'type=gha,scope=${{ github.ref }}_${{ github.repo }}' || '' }}
         platforms: "linux/amd64,linux/arm64"
-        cache-to: type=gha,scope=${{ github.ref }}_${{ github.repo }}
         context: ./
         push: true
         tags: ghcr.io/sirherobrine23/apt-stream:nightly

.github/workflows/publish.yaml

@@ -9,6 +9,11 @@ jobs:
   publishpackage:
     runs-on: ubuntu-latest
     name: Publish
+    permissions:
+      packages: write
+      contents: write
+    env:
+      PACKAGE_VERSION: ${{ github.ref }}
     steps:
     - uses: actions/checkout@v3
       name: Code checkout
@@ -18,39 +23,6 @@ jobs:
         fetch-depth: 2
         submodules: true
 
-    # Install basic tools
-    - uses: actions/setup-node@v3
-      name: Setup node.js
-      with:
-        node-version: 18.x
-        registry-url: https://registry.npmjs.org/
-
-    - run: sudo npm install -g ts-node typescript
-      name: Install typescript and ts-node
-
-    - name: Edit version
-      shell: node {0}
-      run: |
-        const fs = require("fs");
-        const path = require("path");
-        const packagePath = path.join(process.cwd(), "package.json");
-        const package = JSON.parse(fs.readFileSync(packagePath, "utf8"));
-        package.version = "${{ github.ref }}";
-        package.version = package.version.replace(/[A-Za-z_\/]+/, "");
-        fs.writeFileSync(packagePath, JSON.stringify(package, null, 2));
-
-    # Add version to environment variables
-    - name: Add version to environment variables
-      run: |
-        cat package.json | jq -r '.version' > /tmp/version.txt
-        echo "PACKAGE_VERSION=$(cat /tmp/version.txt)" >> $GITHUB_ENV
-
-    # Install depencides and build
-    - run: npm ci
-
-    # Build
-    - run: npm run build
-
     - name: Setup QEMU to Docker
       uses: docker/setup-qemu-action@v2
 
@@ -64,12 +36,32 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
+    # Install basic tools
+    - uses: actions/setup-node@v3
+      name: Setup node.js
+      with:
+        node-version: 18.x
+        registry-url: https://registry.npmjs.org/
+
+    - name: Edit version and install depencies
+      run: |
+        sudo npm i -g semver
+        VERSION="$(semver -c ${{ github.ref_name }})"
+        echo "PACKAGE_VERSION=$VERSION" >> $GITHUB_ENV
+        jq --arg ver $VERSION '.version = $ver' package.json
+
+        # Install depencides and build
+        npm install --no-save
+
+    # Publish npm
+    - run: npm publish --access public --tag ${{ github.event.release.prerelease && 'next' || 'latest' }}
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
     - name: Build image
       uses: docker/build-push-action@v4
       with:
-        cache-from: ${{ github.event_name == 'push' && 'type=gha,scope=${{ github.ref }}_${{ github.repo }}' || '' }}
         platforms: "linux/amd64,linux/arm64"
-        cache-to: type=gha,scope=${{ github.ref }}_${{ github.repo }}
         context: ./
         push: true
         tags: |

.gitignore

@@ -6,6 +6,7 @@ thunder-tests
 *.deb
 *.key
 *.pem
+*.gpg
 
 # Ingore node folders
 node_modules/

.npmignore

@@ -16,12 +16,18 @@ node_modules/
 
 # Docker
 .dockerignore
-.Dockerfile
+*docker-compose.yaml
+*docker-compose.yml
+*Dockerfile*
+*dockerfile*
 
 # Project
-/apt*.y[a]ml
+/apt*.yml
+/apt*.yaml
+/apt*.json
 .repoTest/
 thunder-tests
 *.deb
 *.key
-*.pem
+*.pem
+*.gpg

Dockerfile

@@ -9,7 +9,7 @@ RUN npm run build
 # Clean build
 FROM node:lts
 VOLUME [ "/data" ]
-COPY --from=0 /app/ /app
 WORKDIR /app
+COPY --from=0 /app/ ./
 RUN npm link
-ENTRYPOINT "apt-stream server --cache /data/cache --data /data/data"
+ENTRYPOINT [ "apt-stream", "server", "--cache", "/data/cache" ]

src/aptServer.ts

@@ -150,7 +150,12 @@ export default function main(packageManeger: packageManeger, config: aptStreamCo
     return createPackage(packages, path.resolve("/", path.posix.join(req.baseUrl, req.path), "../../../../.."), req.path.endsWith(".gzip") ? "gzip" : req.path.endsWith(".xz") ? "lzma" : undefined).pipe(res.writeHead(200, {
     }));
   });
-
+  app.get("/pool", async ({res}) => packageManeger.search({}).then(data => res.json(data)));
+  app.get("/pool/:componentName", async (req, res) => {
+    const packagesList = await packageManeger.search({packageComponent: req.params.componentName});
+    if (packagesList.length === 0) return res.status(404).json({error: "Package component not exists"});
+    return res.json(packagesList.map(({packageControl, packageDistribuition}) =>  ({control: packageControl, dist: packageDistribuition})));
+  });
   app.get("/pool/:componentName/(:package)_(:arch)_(:version).deb", async (req, res, next) => {
     const { componentName, package: packageName, arch, version: packageVersion } = req.params;
     const packageID = (await packageManeger.search({packageComponent: componentName, packageArch: arch})).find(({packageControl: { Package, Version }}) => packageName === Package && Version === packageVersion);
@@ -158,6 +163,5 @@ export default function main(packageManeger: packageManeger, config: aptStreamCo
     console.log(packageID);
     return fileRestore(packageID, config).then(str => str.pipe(res.writeHead(200, {}))).catch(next);
   });
-
   return app;
 }

src/config.ts

@@ -155,6 +155,11 @@ export async function prettyConfig(tmpConfig: aptStreamConfig, optionsOverload?:
     repository: {},
   };
 
+  if (newConfigObject.gpgSign) {
+    if (!newConfigObject.gpgSign.private.content && newConfigObject.gpgSign.private.path) newConfigObject.gpgSign.private.content = await fs.readFile(path.resolve(process.cwd(), newConfigObject.gpgSign.private.path), "utf8");
+    if (!newConfigObject.gpgSign.public.content && newConfigObject.gpgSign.public.path) newConfigObject.gpgSign.public.content = await fs.readFile(path.resolve(process.cwd(), newConfigObject.gpgSign.public.path), "utf8");
+  }
+
   for (const repoName of returnUniq((Object.keys(optionsOverload?.repository ?? {}).concat(...(Object.keys(tmpConfig.repository ?? {})))))) {
     for (const data of ((optionsOverload?.repository?.[repoName]?.source ?? []).concat(tmpConfig?.repository?.[repoName]?.source)).filter(Boolean)) {
       if (!data) continue;
@@ -228,10 +233,30 @@ export async function prettyConfig(tmpConfig: aptStreamConfig, optionsOverload?:
   return newConfigObject;
 }
 
+export async function convertString(config: aptStreamConfig, target: "yaml"|"yml"|"json"|"json64"|"yaml64"|"yml64") {
+  config = await prettyConfig(config);
+  let encode64 = target.endsWith("64");
+  let configString: string;
+  if (target === "json"||target === "json64") configString = JSON.stringify(config, null, encode64 ? 0 : 2);
+  else configString = yaml.stringify(config);
+  if (encode64) return Buffer.from(configString, "utf8").toString("base64");
+  return configString;
+}
+
 export async function save(configPath: string, config: aptStreamConfig) {
+  config = await prettyConfig(config);
+  if (config.gpgSign) {
+    if (config.gpgSign.private.path) {
+      await fs.writeFile(path.resolve(process.cwd(), config.gpgSign.private.path), config.gpgSign.private.content);
+      config.gpgSign.private.content = null;
+    }
+    if (config.gpgSign.public.path) {
+      await fs.writeFile(path.resolve(process.cwd(), config.gpgSign.public.path), config.gpgSign.public.content);
+      config.gpgSign.public.content = null;
+    }
+  }
   let ext = ".json";
   if (path.extname(configPath) === ".yaml" || path.extname(configPath) === ".yml") ext = ".yaml";
-  config = await prettyConfig(config);
   return fs.writeFile(configPath, ext === ".json" ? JSON.stringify(config, null, 2) : yaml.stringify(config));
 }
 

src/configManeger.ts

@@ -5,7 +5,11 @@ import { googleDriver, oracleBucket } from "@sirherobrine23/cloud";
 import { syncRepository } from "./packageManege.js";
 import { connect } from "./database.js";
 import { Github } from "@sirherobrine23/http";
+import openpgp from "openpgp";
 import ora from "ora";
+import path from "path";
+import fs from "fs/promises";
+import { extendsFS } from "@sirherobrine23/extends";
 
 async function simpleQuestion<T = any>(promp: QuestionCollection): Promise<Awaited<T>> {
   promp["name"] ??= "No name";
@@ -343,6 +347,87 @@ async function manegerSource(config: aptStreamConfig, repositoryName: string): P
   return config;
 }
 
+async function genGPG(config: aptStreamConfig): Promise<aptStreamConfig> {
+  if (config.gpgSign) console.warn("Replacing exists gpg keys");
+
+  const ask = await inquirer.prompt([
+    {
+      type: "input",
+      message: "Full name or nickname, example Google Inc.:",
+      name: "name",
+    },
+    {
+      type: "input",
+      message: "email, example: noreply@gmail.com:",
+      name: "email"
+    },
+    {
+      type: "password",
+      mask: "*",
+      message: "password to encrypt the gpg files, if you don't want to leave it blank",
+      name: "pass",
+      validate(input = "") {
+        if (input.length === 0) return true;
+        else if (input.length >= 8) return true;
+        return "Password must have more than 8 characters!";
+      },
+    },
+    {
+      type: "password",
+      mask: "*",
+      name: "passConfirm",
+      when: (answers) => answers.pass?.length > 0,
+      validate(input, answers) {
+        if (input === answers.pass) return true;
+        return "Invalid password, check is same!";
+      },
+    },
+    {
+      type: "confirm",
+      message: "Want to save keys locally?",
+      name: "confirmSaveGPG",
+    },
+    {
+      type: "input",
+      message: "Which folder do you save?",
+      name: "folderPath",
+      default: path.resolve(process.cwd(), "gpgKeys"),
+      when: (answers) => answers.confirmSaveGPG
+    }
+  ]);
+  return openpgp.generateKey({
+    rsaBits: 4096,
+    format: "armored",
+    type: "rsa",
+    passphrase: ask.pass,
+    userIDs: [{
+      comment: "Generated by apt-stream",
+      name: ask.name,
+      email: ask.email,
+    }],
+  }).then(async keys => {
+    config.gpgSign = {
+      authPassword: ask.pass,
+      private: {
+        content: keys.privateKey,
+      },
+      public: {
+        content: keys.publicKey,
+      }
+    };
+    if (ask.confirmSaveGPG) {
+      const folderPath = path.resolve(process.cwd(), ask.folderPath);
+      if (!(await extendsFS.exists(folderPath))) await fs.mkdir(folderPath, {recursive: true});
+      config.gpgSign.private.path = path.join(folderPath, "privateAptStream.gpg");
+      config.gpgSign.public.path = path.join(folderPath, "publicAptStream.gpg");
+    }
+    return config;
+  }).catch(err => {
+    console.error(err?.message || err);
+    return genGPG(config);
+  });
+}
+
 export default async function main(configPath: string, configOld?: aptStreamConfig) {
   if (configOld) {
     console.log("Saving current config...");
@@ -353,18 +438,20 @@ export default async function main(configPath: string, configOld?: aptStreamConf
     console.log("Init fist repository config!");
     return createRepository(localConfig).then(d => main(configPath, d));
   }
-  const target = await simpleQuestion<"new"|"edit"|"load"|"exit">({
+  const target = await simpleQuestion<"new"|"gpg"|"edit"|"load"|"exit">({
     type: "list",
     message: "Select action",
     choices: [
       {name: "Edit repository", value: "edit"},
       {name: "Create new Repository", value: "new"},
+      {name: "(Re)generate gpg keys", value: "gpg"},
       {name: "Sync repository", value: "load"},
       {name: "Exit", value: "exit"}
     ]
   });
   if (target !== "exit") {
     if (target === "new") configOld = await createRepository(localConfig);
+    if (target === "gpg") configOld = await genGPG(localConfig);
     else if (target === "edit") {
       const repoName = await simpleQuestion<string>({
         type: "list",

src/database.ts

@@ -24,7 +24,7 @@ export class packageManeger {
     return this.options.getPackages.call(this);
   }
 
-  search = async (search: {packageName?: string, packageArch?: string, packageComponent?: string, packageDist?: string}): ReturnType<typeof this.options.findPackages> => {
+  async search(search: {packageName?: string, packageArch?: string, packageComponent?: string, packageDist?: string}): Promise<packageData[]> {
     if (typeof this.options.findPackages !== "function") return (await this.getPackages()).filter(data => ((!search.packageName) || (search.packageName === data.packageControl.Package)) && ((!search.packageArch) || (data.packageControl.Architecture === search.packageArch)) && ((!search.packageComponent) || (data.packageComponent === search.packageComponent)) && ((!search.packageDist) || (data.packageDistribuition === search.packageDist)));
     return this.options.findPackages.call(this, search);
   }

src/index.ts

@@ -1,15 +1,15 @@
 #!/usr/bin/env node
 import "./log.js";
 import { connect } from "./database.js";
-import { config } from "./config.js";
+import { config, convertString } from "./config.js";
 import packageManeger from "./configManeger.js";
 import express from "express";
 import yargs from "yargs";
 import cluster from "node:cluster";
 import apt from "./aptServer.js";
+import openpgp from "openpgp";
 
-yargs(process.argv.slice(2)).version(false).help(true).strictCommands().demandCommand().alias("h", "help").command("server", "Run http Server", async yargs => {
-  const options = yargs.option("config", {
+yargs(process.argv.slice(2)).version(false).help(true).strictCommands().demandCommand().alias("h", "help").command("server", "Run http Server", yargs => yargs.option("config", {
     string: true,
     alias: "c",
     type: "string",
@@ -30,7 +30,7 @@ yargs(process.argv.slice(2)).version(false).help(true).strictCommands().demandCo
     alias: "C",
     type: "string",
     description: "cache files"
-  }).parseSync();
+  }), async options => {
   const appConfig = await config(options.config, {serverConfig: {portListen: options.port, clusterCount: options.cluster, cacheFolder: options.cache}});
   if ((appConfig.serverConfig?.clusterCount || 0) > 0 && cluster.isPrimary) {
     const ct = () => {
@@ -47,19 +47,29 @@ yargs(process.argv.slice(2)).version(false).help(true).strictCommands().demandCo
   }
   const db = await connect(appConfig);
   const app = express();
+  app.get("/", ({res}) => res.json({cluster: cluster.isWorker, id: cluster.worker?.id}));
+  app.get("/public(_key|)(|.gpg)", async ({res}) => {
+    if (!appConfig.gpgSign) return res.status(404).json({error: "Gpg not configured"});
+    const pubKey = (await openpgp.readKey({ armoredKey: appConfig.gpgSign.public.content })).armor();
+    return res.setHeader("Content-Type", "application/pgp-keys").send(pubKey);
+  });
   const aptRoute = apt(db, appConfig);
   app.use(aptRoute);
   app.listen(appConfig.serverConfig?.portListen ?? 0, function () {
     const address = this.address();
     console.log("Port Listen on %O", typeof address === "object" ? address.port : address);
   });
-}).command("package", "maneger packages in database", yargs => {
-  const { config } = yargs.option("config", {
+}).command(["maneger", "m", "$0"], "maneger packages in database", yargs => {
+  return yargs.option("config", {
     string: true,
     alias: "c",
     type: "string",
     description: "Config file path",
-    default: "aptStream.yml"
-  }).parseSync();
-  return packageManeger(config);
+    default: "aptStream.yml",
+  }).command(["$0"], "Maneger config", yargs => yargs, options => packageManeger(options.config)).command(["print", "p"], "Print config to target default is json", yargs => yargs.option("outputType", {
+    alias: "o",
+    choices: ["yaml", "yml", "json", "json64", "yaml64", "yml64"],
+    description: "target output file, targets ended with '64' is base64 string",
+    default: "json"
+  }), async (options) => console.log(await convertString(await config(options.config), options.outputType as any)));
 }).parseAsync();

src/log.ts

@@ -10,7 +10,7 @@ if (cluster.isWorker) {
     depth: null
   };
 
-  console.clear = console.clear ?? function () {console.warn("Not tty")}
+  console.clear = console.clear ?? function () {console.warn("cannot clear tty");}
 
   console.log = function(...args) {
     log("[LOG%s]: %s", id ? ` Cluster ${id}` : "", formatWithOptions(defaultOptions, ...args));