rewrite API #3

Merged
Sirherobrine23 merged 6 commits from v2APIs into main 2022-12-24 02:00:38 +00:00
16 changed files with 760 additions and 514 deletions
Showing only changes of commit 2c028a8d40 - Show all commits

6
.dockerignore Normal file
View File

@ -0,0 +1,6 @@
node_modules/
.devcontainer/
.vscode/
.github/
*.d.ts
*.js

9
Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM nodejs:latest
VOLUME [ "/data" ]
WORKDIR /app
COPY package*.json ./
RUN npm ci
COPY . .
RUN npm run build
EXPOSE 3000
ENTRYPOINT [ "node", "src/index.js", "--port", "3000", "--config-path", "/data/.apt_stream.yml" ]

View File

@ -1,18 +1,52 @@
# node-apt
# apt-stream
Crie seu repositorio apt com o nodejs sem precisas salvar arquivos ou mesmo cuidar do armazenamento.
Create your apt repository with nodejs without having to save files or even take care of storage.
## Storages
Como esse projeto voçe hospedar um repositorio do apt rapida com os seguintes storages:
You can host an apt rapida repository with the following storages:
- Docker and OCI images (find `.deb` files in diff's)
- Github Releases
- Local folders
## Config file
Estou ainda mexendo com o arquivo de configuração dos repositorios, e do servidor por enquanto está com está até eu poder mexer direito nele.
```yaml
# Global apt config
apt-config:
origin: ""
enableHash: true # if it is enabled, it may freeze the request a little because I have to wait for the hashes of the "Packages" files
sourcesHost: http://localhost:3000
# If you want to use a custom sources.list
sourcesList: deb [trusted=yes] %s://%s %s main
repositories:
# Example to docker and OCI image
- from: oci
image: ghcr.io/sirherobrine23/nodeaptexample
# Release endpoint config
apt-config:
origin: github.com/cli/cli
lebel: github-cli
description: |
This is example
is this second line of description.
# Example to github release
- from: github_release
repository: cli/cli
- from: github_release
owner: cli
repository: cli
token: ""
```
## Endpoints
Esse projeto foi feito com base nas rotas do `archive.ubuntu.com` e do `ftp.debian.org`.
This project was made based on the `archive.ubuntu.com` and `ftp.debian.org` routes.
* `GET` /dists/:package-name/Release
* `GET` /dists/:package-name/main/binary-:arch/Packages
@ -22,7 +56,4 @@ Esse projeto foi feito com base nas rotas do `archive.ubuntu.com` e do `ftp.debi
### Download .deb and package info
* `GET` /pool/:package_name/:version/:arch.deb - `Download .deb file`
* `GET` /pool/:package_name/:version/:arch - `Config package if exists`
* `GET` /pool/:package_name/:version - `Get version with config`
* `GET` /pool/:package_name - `Get all versions with config`
* `GET` /pool - `Get packages registred to packages registry`
* `GET` /pool and / - `Get packages registred to packages registry`

70
package-lock.json generated
View File

@ -1,22 +1,27 @@
{
"name": "node-apt",
"name": "apt-stream",
"version": "1.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "node-apt",
"name": "apt-stream",
"version": "1.0.0",
"license": "GPL-2.0",
"dependencies": {
"@sirherobrine23/coreutils": "^2.1.6",
"cron": "^2.1.0",
"express": "^4.18.2",
"lzma-native": "^8.0.6",
"tar": "^6.1.13",
"yaml": "^2.1.3",
"yargs": "^17.6.2"
},
"bin": {
"apt-stream": "src/index.js"
},
"devDependencies": {
"@types/cron": "^2.0.0",
"@types/express": "^4.17.15",
"@types/lzma-native": "^4.0.1",
"@types/node": "^18.11.17",
@ -272,6 +277,16 @@
"@types/node": "*"
}
},
"node_modules/@types/cron": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz",
"integrity": "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ==",
"dev": true,
"dependencies": {
"@types/luxon": "*",
"@types/node": "*"
}
},
"node_modules/@types/express": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.15.tgz",
@ -300,6 +315,12 @@
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz",
"integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ=="
},
"node_modules/@types/luxon": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.1.0.tgz",
"integrity": "sha512-gCd/HcCgjqSxfMrgtqxCgYk/22NBQfypwFUG7ZAyG/4pqs51WLTcUzVp1hqTbieDYeHS3WoVEh2Yv/2l+7B0Vg==",
"dev": true
},
"node_modules/@types/lzma-native": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@types/lzma-native/-/lzma-native-4.0.1.tgz",
@ -642,6 +663,14 @@
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"dev": true
},
"node_modules/cron": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz",
"integrity": "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w==",
"dependencies": {
"luxon": "^1.23.x"
}
},
"node_modules/cssom": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
@ -1297,6 +1326,14 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==",
"engines": {
"node": "*"
}
},
"node_modules/lzma-native": {
"version": "8.0.6",
"resolved": "https://registry.npmjs.org/lzma-native/-/lzma-native-8.0.6.tgz",
@ -2468,6 +2505,16 @@
"@types/node": "*"
}
},
"@types/cron": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz",
"integrity": "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ==",
"dev": true,
"requires": {
"@types/luxon": "*",
"@types/node": "*"
}
},
"@types/express": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.15.tgz",
@ -2496,6 +2543,12 @@
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz",
"integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ=="
},
"@types/luxon": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.1.0.tgz",
"integrity": "sha512-gCd/HcCgjqSxfMrgtqxCgYk/22NBQfypwFUG7ZAyG/4pqs51WLTcUzVp1hqTbieDYeHS3WoVEh2Yv/2l+7B0Vg==",
"dev": true
},
"@types/lzma-native": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@types/lzma-native/-/lzma-native-4.0.1.tgz",
@ -2776,6 +2829,14 @@
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"dev": true
},
"cron": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz",
"integrity": "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w==",
"requires": {
"luxon": "^1.23.x"
}
},
"cssom": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
@ -3262,6 +3323,11 @@
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
"integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ=="
},
"luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
},
"lzma-native": {
"version": "8.0.6",
"resolved": "https://registry.npmjs.org/lzma-native/-/lzma-native-8.0.6.tgz",

View File

@ -1,21 +1,21 @@
{
"name": "node-apt",
"name": "apt-stream",
"version": "1.0.0",
"description": "Replace",
"main": "src/index.js",
"types": "./src/index.d.ts",
"private": false,
"type": "module",
"homepage": "https://github.com/Sirherobrine23/node-apt#readme",
"homepage": "https://github.com/Sirherobrine23/apt-stream#readme",
"author": "Matheus Sampaio Queiroga <srherobrine20@gmail.com> (https://sirherobrine23.org/)",
"license": "GPL-2.0",
"repository": {
"type": "git",
"url": "git+https://github.com/Sirherobrine23/node-apt.git"
"url": "git+https://github.com/Sirherobrine23/apt-stream.git"
},
"keywords": [],
"bugs": {
"url": "https://github.com/Sirherobrine23/node-apt/issues"
"url": "https://github.com/Sirherobrine23/apt-stream/issues"
},
"sponsor": {
"url": "https://github.com/sponsors/Sirherobrine23"
@ -26,10 +26,15 @@
"engines": {
"node": ">=14.0.0"
},
"bin": {
"apt-stream": "./src/index.js"
},
"scripts": {
"start": "ts-node src/index.ts"
"start": "ts-node src/index.ts",
"build": "tsc"
},
"devDependencies": {
"@types/cron": "^2.0.0",
"@types/express": "^4.17.15",
"@types/lzma-native": "^4.0.1",
"@types/node": "^18.11.17",
@ -40,6 +45,7 @@
},
"dependencies": {
"@sirherobrine23/coreutils": "^2.1.6",
"cron": "^2.1.0",
"express": "^4.18.2",
"lzma-native": "^8.0.6",
"tar": "^6.1.13",

View File

@ -1,13 +1,22 @@
version: 1
repos: []
# - repo: ghcr.io/sirherobrine23/nodeaptexample:sha256:c7b3eb9435f67aa08a2e98fdaa5c4548bb8e410fc804ed352bb03dcb212dde96
# from: oci
# Global apt config
apt-config:
origin: ""
enableHash: true # if it is enabled, it may freeze the request a little because I have to wait for the hashes of the "Packages" files
sourcesHost: http://localhost:3000
# If you want to use a custom sources.list
sourcesList: deb [trusted=yes] %s://%s %s main
# - repo: Sirherobrine23/ghcrAptRepository
# from: release
repositories:
# Example to docker and OCI image
# - from: oci
# image: ghcr.io/sirherobrine23/nodeaptexample
# # Release endpoint config
# apt-config:
# origin: github.com/cli/cli
# lebel: github-cli
# - repo: The-Bds-Maneger/BedrockFetch
# from: release
# - repo: cli/cli
# from: release
# Example to github release
- from: github_release
repository: cli
owner: cli
takeUpTo: 2

View File

@ -1,335 +0,0 @@
import { format } from "node:util";
import { WriteStream } from "node:fs";
import { PassThrough, Readable, Writable } from "node:stream";
import { getConfig } from "./repoConfig.js";
import * as ghcr from "./oci_registry.js";
import * as release from "./githubRelease.js";
import zlib from "node:zlib";
import express from "express";
import coreUtils from "@sirherobrine23/coreutils";
export type packagesObject = {
Package: string
Version: string,
/** endpoint folder file */
Filename: string,
"Installed-Size": number,
Maintainer: string,
Architecture: string,
Depends?: string,
Homepage?: string,
Section?: string,
Priority?: string,
Size: number,
MD5sum: string,
SHA256: string,
Description?: string,
};
export type ReleaseOptions = {
Origin?: string,
Suite?: string,
Archive?: string,
lebel?: string,
Codename?: string,
Architectures: string[],
Components: string[],
Description?: string,
sha256?: {sha256: string, size: number, file: string}[]
};
export function parseDebControl(control: string|Buffer) {
if (Buffer.isBuffer(control)) control = control.toString();
const controlObject: {[key: string]: string} = {};
for (const line of control.split(/\r?\n/)) {
if (/^[\w\S]+:/.test(line)) {
const [, key, value] = line.match(/^([\w\S]+):(.*)$/);
controlObject[key.trim()] = value.trim();
} else {
controlObject[Object.keys(controlObject).at(-1)] += line;
}
}
return controlObject;
}
export function mountRelease(repo: ReleaseOptions) {
let data = [`Lebel: ${repo.lebel||repo.Origin}`];
data.push(`Date: ${(new Date()).toUTCString()}`);
if (repo.Origin) data.push(`Origin: ${repo.Origin}`);
if (repo.Suite) data.push(`Suite: ${repo.Suite}`);
else if (repo.Archive) data.push(`Archive: ${repo.Archive}`);
if (repo.Codename) data.push(`Codename: ${repo.Codename}`);
data.push(`Architectures: ${repo.Architectures.join(" ")}\nComponents: ${repo.Components.join(" ")}`);
if (repo.Description) data.push(`Description: ${repo.Description}`);
if (repo.sha256 && repo.sha256?.length > 0) {
data.push("SHA256:");
for (const file of repo.sha256) {
data.push(` ${file.sha256} ${file.size} ${file.file}`);
}
}
return data.join("\n")+"\n";
}
export type registryPackageData = {
name: string,
getStrem: () => Promise<Readable>,
version: string,
arch: string,
size?: number,
from?: string,
signature?: {
sha256: string,
md5: string,
},
packageConfig?: {
[key: string]: string;
}
};
type localRegister = {
[name: string]: {
[version: string]: {
[arch: string]: {
getStream: registryPackageData["getStrem"],
config?: registryPackageData["packageConfig"],
signature?: registryPackageData["signature"],
size?: registryPackageData["size"],
from?: registryPackageData["from"],
}
}
}
};
export class localRegistryManeger {
public packageRegister: localRegister = {};
prettyPackages() {
const packagePretty = {};
for (const name in this.packageRegister) {
if (!packagePretty[name]) packagePretty[name] = [];
for (const version in this.packageRegister[name]) {
for (const arch in this.packageRegister[name][version]) {
packagePretty[name].push({
version,
arch,
config: this.packageRegister[name][version][arch].config,
});
}
}
}
return packagePretty;
}
public registerPackage(packageConfig: registryPackageData) {
packageConfig.name = packageConfig.name?.toLowerCase()?.trim();
packageConfig.version = packageConfig?.version?.trim();
if (!this.packageRegister) this.packageRegister = {};
if (!this.packageRegister[packageConfig.name]) this.packageRegister[packageConfig.name] = {};
if (!this.packageRegister[packageConfig.name][packageConfig.version]) this.packageRegister[packageConfig.name][packageConfig.version] = {};
console.log("[Internal package maneger]: Registry %s with version %s and arch %s", packageConfig.name, packageConfig.version, packageConfig.arch);
this.packageRegister[packageConfig.name][packageConfig.version][packageConfig.arch] = {
getStream: packageConfig.getStrem,
config: packageConfig.packageConfig,
signature: packageConfig.signature,
size: packageConfig.size,
};
}
public async createPackage(compress: boolean, name: string, arch: string, res: WriteStream|Writable|PassThrough = new PassThrough({read(){}, write(){}})) {
const sign: {sha256?: {hash: string, size: number}, md5?: {hash: string, size: number}, size: number} = {size: 0};
const Packages: packagesObject[] = [];
if (!this.packageRegister[name]) throw new Error(format("%s not found", name));
for (const version in this.packageRegister[name]) {
const packageData = this.packageRegister[name][version][arch];
if (!packageData) continue;
Packages.push({
Package: name,
Version: version,
Filename: format("pool/%s/%s/%s.deb", name, version, arch),
"Installed-Size": parseInt(packageData.config?.["Installed-Size"]||"0"),
Maintainer: packageData.config?.Maintainer||"node-apt",
Architecture: packageData.config?.Architecture||"all",
Depends: packageData.config?.Depends,
Homepage: packageData.config?.Homepage,
Section: packageData.config?.Section,
Priority: packageData.config?.Priority,
Size: packageData.size||0,
MD5sum: packageData.signature?.md5,
SHA256: packageData.signature?.sha256,
Description: packageData.config?.Description,
});
}
let vsize = 0;
const rawStream = new PassThrough({read(){}, write(){}});
if (compress) {
const ReadStream = new PassThrough({read(){}, write(){}});
const gzip = ReadStream.pipe(zlib.createGzip());
gzip.pipe(res);
gzip.pipe(rawStream);
gzip.on("data", (chunk) => vsize += chunk.length);
res = ReadStream;
} else rawStream.on("data", (chunk) => vsize += chunk.length);
let waitHashs = coreUtils.extendsCrypto.createSHA256_MD5(rawStream, "both", new Promise(done => {
rawStream.on("end", () => setTimeout(done, 500));
rawStream.on("close", () => setTimeout(done, 500));
})).then(data => {
sign.sha256 = {hash: data.sha256, size: vsize};
sign.md5 = {hash: data.md5, size: vsize};
});
for (const packageInfo of Packages) {
let packageData = [];
for (let i in packageInfo) packageData.push(`${i}: ${packageInfo[i]||""}`);
const configLine = packageData.join("\n")+"\n\n";
sign.size += configLine.length;
rawStream.push(Buffer.from(configLine, "utf8"));
rawStream.write(Buffer.from(configLine, "utf8"));
if (res instanceof PassThrough) res.push(configLine);
else res.write(configLine);
}
res.end();
res.destroy();
rawStream.end();
rawStream.destroy();
await waitHashs;
return sign;
}
}
async function mainConfig(configPath: string) {
const config = await getConfig(configPath);
const packageReg = new localRegistryManeger();
Promise.all(config.repos.map(async repo => {
if (repo.from === "release") return release.fullConfig({config: repo.repo, githubToken: repo?.auth?.password}, () => {}).catch(console.error);
if (repo.from === "oci") return ghcr.fullConfig({image: repo.repo, targetInfo: repo.ociConfig}, () => {}).catch(console.error);
}));
return packageReg;
}
export type apiConfig = {
configPath: string,
portListen?: number,
callback?: (port: number) => void,
repositoryOptions?: {
Origin?: string,
lebel?: string,
}
};
export async function createAPI(apiConfig: apiConfig) {
const mainRegister = await mainConfig(apiConfig.configPath);
const app = express();
app.use(express.json());
app.use(express.urlencoded({extended: true}));
app.use((_req, res, next) => {
res.json = (data) => res.setHeader("Content-Type", "application/json").send(JSON.stringify(data, null, 2));
next();
});
// Request log
app.use((req, _res, next) => {
next();
console.log("[%s]: From: %s, path: %s", req.protocol, req.ip, req.path);
});
// /dists
// Signed Release with gpg
// app.get("/dists/:suite/InRelease", (req, res) => {});
// app.get("/dists/:package/Release.gpg", (req, res) => {});
// Root release
app.get("/dists/:package/Release", async (req, res) => {
if (!mainRegister.packageRegister[req.params.package]) return res.status(404).json({error: "Package not registred"});
const Archs: string[] = [];
Object.keys(mainRegister.packageRegister[req.params.package]).forEach(version => Object.keys(mainRegister.packageRegister[req.params.package][version]).forEach(arch => (!Archs.includes(arch.toLowerCase()))?Archs.push(arch.toLowerCase()):null));
const shas: ReleaseOptions["sha256"] = [];
for (const arch of Archs) {
const Packagegz = await mainRegister.createPackage(true, req.params.package, arch);
const Package = await mainRegister.createPackage(false, req.params.package, arch);
if (Packagegz?.sha256?.hash) shas.push({file: format("main/binary-%s/Packages.gz", arch), sha256: Packagegz.sha256.hash, size: Packagegz.sha256.size});
if (Package?.sha256?.hash) shas.push({file: format("main/binary-%s/Packages", arch), sha256: Package.sha256.hash, size: Package.sha256.size});
}
const data = mountRelease({
Origin: apiConfig.repositoryOptions?.Origin||"node-apt",
lebel: apiConfig.repositoryOptions?.lebel,
Suite: req.params.package,
Components: ["main"],
Architectures: Archs,
sha256: shas
});
res.setHeader("Content-Type", "text/plain");
res.setHeader("Content-Length", data.length);
return res.send(data);
});
// Binary release
app.get("/dists/:package/main/binary-:arch/Release", (req, res) => {
const Archs: string[] = [];
Object.keys(mainRegister.packageRegister[req.params.package]).forEach(version => Object.keys(mainRegister.packageRegister[req.params.package][version]).forEach(arch => (!Archs.includes(arch.toLowerCase()))?Archs.push(arch.toLowerCase()):null));
if (!Archs.includes(req.params.arch.toLowerCase())) return res.status(404).json({error: "Package arch registred"});
res.setHeader("Content-Type", "text/plain");
return res.send(mountRelease({
Origin: apiConfig.repositoryOptions?.Origin||"node-apt",
lebel: apiConfig.repositoryOptions?.lebel,
Archive: req.params.package,
Components: ["main"],
Architectures: [req.params.arch.toLowerCase()]
}));
});
// binary packages
app.get("/dists/:package/main/binary-:arch/Packages(.gz|)", (req, res, next) => mainRegister.createPackage(req.path.endsWith(".gz"), req.params.package, req.params.arch, res.writeHead(200, {"Content-Type": req.path.endsWith(".gz") ? "application/x-gzip" : "text/plain"})).catch(next));
// source.list
app.get("/*.list", (req, res) => {
res.setHeader("Content-type", "text/plain");
let config = "";
Object.keys(mainRegister.packageRegister).forEach(packageName => config += format("deb [trusted=yes] %s://%s %s main\n", req.protocol, req.headers.host, packageName));
res.send(config+"\n");
});
// Pool
app.get(["/", "/pool"], (_req, res) => res.json(mainRegister.prettyPackages()));
app.get("/pool/:package_name", (req, res) => {
const {package_name} = req.params;
const info = mainRegister.packageRegister[package_name];
if (!info) return res.status(404).json({error: "Package not registred"});
return res.json(info);
});
app.get("/pool/:package_name/:version", (req, res) => {
const {package_name, version} = req.params;
const info = mainRegister.packageRegister[package_name];
if (!info) return res.status(404).json({error: "Package not registred"});
const ver = info?.[version];
if (!ver) return res.status(404).json({error: "version not registred"});
return res.json(ver);
});
app.get("/pool/:package_name/:version/:arch.deb", (req, res) => {
const {package_name, arch, version} = req.params;
const info = mainRegister.packageRegister[package_name];
if (!info) return res.status(404).json({error: "Package not registred"});
const ver = info?.[version];
if (!ver) return res.status(404).json({error: "version not registred"});
const archInfo = ver?.[arch];
if (!archInfo) return res.status(404).json({error: "arch not registred"});
const stream = archInfo?.getStream;
if (!stream) return res.status(404).json({error: "Package not registred"});
res.writeHead(200, {"Content-Type": "application/x-debian-package"});
return Promise.resolve(stream()).then(stream => stream.pipe(res));
});
app.get("/pool/:package_name/:version/:arch", (req, res) => {
const {package_name, arch, version} = req.params;
const info = mainRegister.packageRegister[package_name];
if (!info) return res.status(404).json({error: "Package not registred"});
const ver = info?.[version];
if (!ver) return res.status(404).json({error: "version not registred"});
const archInfo = ver?.[arch];
if (!archInfo) return res.status(404).json({error: "arch not registred"});
return res.json(archInfo);
});
app.listen(apiConfig.portListen||0, function listen() {
if (typeof apiConfig.callback !== "function") apiConfig.callback = (port) => console.log("API listen on port %s", port);
apiConfig.callback(this.address().port);
});
return app;
}

View File

@ -19,7 +19,7 @@ export function packageManeger(RootOptions?: backendConfig) {
const localRegister: registerOobject = {};
function pushPackage(control: packageControl, getStream: () => Promise<Readable>, from?: string) {
if (!localRegister[control.Package]) localRegister[control.Package] = [];
console.log("Register %s/%s-5s", control.Package, control.Version, control.Architecture);
// console.log("Register %s/%s-5s", control.Package, control.Version, control.Architecture);
localRegister[control.Package].push({
getStream,
control,
@ -124,9 +124,9 @@ export function packageManeger(RootOptions?: backendConfig) {
async function createRelease(options?: {packageName?: string, Arch?: string, includesHashs?: boolean, component?: string[], archive?: string}) {
const textLines = [];
if (RootOptions?.aptConfig?.origin) textLines.push(`Origin: ${RootOptions.aptConfig.origin}`);
if (RootOptions?.["apt-config"]?.origin) textLines.push(`Origin: ${RootOptions["apt-config"].origin}`);
if (options?.archive) textLines.push(`Archive: ${options?.archive}`);
else textLines.push(`Lebel: ${RootOptions?.aptConfig?.label||"node-apt"}`, `Date: ${new Date().toUTCString()}`);
else textLines.push(`Lebel: ${RootOptions?.["apt-config"]?.label||"node-apt"}`, `Date: ${new Date().toUTCString()}`);
const components = options?.component ?? ["main"];
const archs: string[] = [];
@ -190,7 +190,7 @@ export default async function repo(aptConfig?: backendConfig) {
const app = express();
const registry = packageManeger();
app.disable("x-powered-by").disable("etag").use(express.json()).use(express.urlencoded({extended: true})).use((_req, res, next) => {
res.json = (data) => res.setHeader("Content-Type", "application/json").send(JSON.stringify(data, null, 2));
res.json = (data) => res.setHeader("Content-Type", "application/json").send(JSON.stringify(data ?? null, null, 2));
next();
}).use((req, _res, next) => {
next();
@ -223,7 +223,7 @@ export default async function repo(aptConfig?: backendConfig) {
if (!Targets) Targets = ["all"];
res.setHeader("Content-type", "text/plain");
let config = "";
for (const target of Targets) config += format(aptConfig?.aptConfig?.sourcesList ?? "deb [trusted=yes] %s://%s %s main\n", req.protocol, host, target);
for (const target of Targets) config += format(aptConfig?.["apt-config"]?.sourcesList ?? "deb [trusted=yes] %s://%s %s main\n", req.protocol, host, target);
res.send(config+"\n");
});
@ -244,7 +244,7 @@ export default async function repo(aptConfig?: backendConfig) {
const { suite, arch } = req.params;
res.setHeader("Content-Type", "text/plain");
return registry.createRelease({
includesHashs: aptConfig?.aptConfig?.enableHash ?? true,
includesHashs: aptConfig?.["apt-config"]?.enableHash ?? true,
Arch: arch,
packageName: suite === "all" ? undefined : suite,
component: [req.params.component],

View File

@ -46,7 +46,7 @@ export function createExtract(fn: (info: fileInfo, stream: Readable) => void) {
}
}
size -= chunk.length;
entryStream.push(chunk, "binary");
if (entryStream) entryStream.push(chunk, "binary");
return callback();
}
let waitMore: Buffer;
@ -64,7 +64,7 @@ export function createExtract(fn: (info: fileInfo, stream: Readable) => void) {
callback();
}
if (!chunk.subarray(0, 8).toString().trim().startsWith("!<arch>")) {
this.destroy();
__writed.destroy();
return callback(new Error("Not an ar file"));
}
__locked = true;

View File

@ -39,7 +39,7 @@ export function parseControl(rawControlFile: string) {
export type debReturn = Awaited<ReturnType<typeof extractDebControl>>;
export async function extractDebControl(debStream: Readable) {
export async function extractDebControl(debStream: Readable, endPromise: Promise<void> = new Promise(done => debStream.once("end", done))) {
return new Promise<{size: number, control: packageControl}>((done, reject) => {
let fileSize = 0;
debStream.on("data", (chunk) => fileSize += chunk.length);
@ -53,7 +53,7 @@ export async function extractDebControl(debStream: Readable) {
controlEntry.on("data", chunck => controlFile = (!controlFile)?chunck:Buffer.concat([controlFile, chunck])).once("end", async () => {
const sign = await signs;
const control = parseControl(controlFile.toString());
debStream.on("end", () => {
endPromise.then(() => {
control.MD5sum = sign.md5;
control.SHA256 = sign.sha256;
control.Size = fileSize;

View File

@ -18,14 +18,14 @@ export async function fullConfig(config: {config: string|baseOptions<{releaseTag
};
}).filter(({assets}) => assets?.length > 0);
for (const rel of releases) {
for (const asset of rel.assets) {
const getStream = async () => coreUtils.httpRequest.pipeFetch(asset.download)
const control = await extractDebControl(await getStream());
fn({
...control,
getStream,
});
}
}
return Promise.all(releases.map(async (rel) => Promise.all(rel.assets.map(async (asset) => {
console.log(`Downloading ${asset.name} from ${rel.tag}`);
const getStream = async () => coreUtils.httpRequest.pipeFetch(asset.download)
const control = await getStream().then(Stream => extractDebControl(Stream, new Promise(done => Stream.once("end", done))));
fn({
...control,
getStream,
});
return {asset, getStream, control};
})))).then(data => data.flat());
}

View File

@ -4,6 +4,7 @@ import repo from "./apt_repo_v2.js";
import { getConfig } from "./repoConfig.js";
import github_release from "./githubRelease.js";
import oci_registry from "./oci_registry.js";
import { CronJob } from "cron";
yargs(process.argv.slice(2)).wrap(null).strict().help().option("cofig-path", {
type: "string",
@ -12,12 +13,34 @@ yargs(process.argv.slice(2)).wrap(null).strict().help().option("cofig-path", {
type: "number",
default: 3000,
}).parseAsync().then(async options => {
const { app, registry } = await repo();
app.listen(options.port, () => {
console.log(`Server listening on port ${options.port}`);
});
Promise.all((await getConfig(options["cofig-path"])).repos.map(async repo => {
if (repo.from === "oci") return oci_registry({image: repo.repo, targetInfo: repo.ociConfig}, data => registry.pushPackage(data.control, data.getStream)).catch(console.error);
else if (repo.from === "release") return github_release({config: repo.repo, githubToken: repo.auth.password}, data => registry.pushPackage(data.control, data.getStream)).catch(console.error);
})).catch(console.error);
const config = await getConfig(options["cofig-path"]);
const { app, registry } = await repo(config);
app.listen(options.port, () => console.log(`Server listening on port ${options.port}`));
for (const repo of config.repositories) {
console.log(repo);
const sen = () => Promise.resolve().then(async () => {
if (repo.from === "github_release") {
const tags = repo.tags ?? ["latest"];
if (tags.length === 0) tags.push("latest");
return Promise.all(tags.map(async tag => github_release({
githubToken: repo.token,
config: {
owner: repo.owner,
repo: repo.repository,
releaseTag: tag,
}
}, ({control, getStream}) => registry.pushPackage(control, getStream))));
} else if (repo.from === "oci") {
return oci_registry({
image: repo.image,
targetInfo: repo.platfom_target,
}, ({control, getStream}) => registry.pushPackage(control, getStream));
}
return null;
}).catch(console.trace);
sen();
const cronJobs = repo.cronRefresh ?? [];
const jobs = cronJobs.map(cron => new CronJob(cron, sen));
jobs.forEach(job => job.start());
}
});

View File

@ -18,11 +18,16 @@ export async function fullConfig(imageInfo: {image: string, targetInfo?: DockerR
return fn({
...control,
getStream: async () => {
return new Promise<Readable>((done, reject) => registry.blobLayerStream(data.layer.digest).then(stream => stream.pipe(tar.list({
onentry(getEntry) {
if (getEntry.path === entry.path) return done(getEntry as any);
}
}))).catch(reject));
return new Promise<Readable>((done, reject) => registry.blobLayerStream(data.layer.digest).then(stream => {
stream.on("error", reject);
stream.pipe(tar.list({
onentry(getEntry) {
if (getEntry.path !== entry.path) return null;
return done(getEntry as any);
}
// @ts-ignore
}).on("error", reject));
}).catch(reject));
},
});
},

View File

@ -2,58 +2,74 @@ import coreUtils, { DockerRegistry } from "@sirherobrine23/coreutils";
import * as yaml from "yaml";
import fs from "node:fs/promises";
export type configV1 = {
version: 1,
repos: (({
from: "release",
repo: string|{
owner: string,
repo: string
},
}|{
from: "oci",
repo: string,
ociConfig?: DockerRegistry.Manifest.platfomTarget,
}) & {
auth?: {
username?: string,
password?: string
}
})[]
export type apt_config = {
origin?: string,
label?: string,
enableHash?: boolean,
sourcesList?: string
};
export type backendConfig = {
aptConfig?: {
origin?: string,
label?: string,
enableHash?: boolean,
sourcesList?: string
export type repository = ({
from: "oci",
image: string,
platfom_target?: DockerRegistry.Manifest.platfomTarget
auth?: {
username?: string,
password?: string
},
repositorys: {
target: "oci_registry"|"github_release",
repo: string,
}[]
};
export async function getConfig(filePath: string): Promise<configV1> {
if (!await coreUtils.extendFs.exists(filePath)) throw new Error("file not exists");
const configData: configV1 = yaml.parse(await fs.readFile(filePath, "utf8"));
return {
version: 1,
repos: (configData?.repos ?? []).map(data => {
if (data.from === "oci" && typeof data.repo === "string") {
return {
repo: data.repo,
from: "oci",
auth: data.auth,
ociConfig: data.ociConfig
};
}
return {
repo: (typeof data.repo === "string")?data.repo:{owner: data.repo.owner, repo: data.repo.repo},
from: "release",
auth: data.auth,
}
})
};
"apt-config"?: apt_config,
}|{
from: "github_release",
repository: string,
owner?: string,
tags?: string[],
takeUpTo?: number,
token?: string,
"apt-config"?: apt_config,
}) & {
/** cron range: https://github.com/kelektiv/node-cron#cron-ranges */
cronRefresh?: string[],
}
export type backendConfig = Partial<{
"apt-config"?: apt_config,
repositories: repository[]
}>;
export async function getConfig(filePath: string) {
if (!await coreUtils.extendFs.exists(filePath)) throw new Error("config File not exists");
const fixedConfig: backendConfig = {};
const configData: backendConfig = yaml.parse(await fs.readFile(filePath, "utf8"));
fixedConfig["apt-config"] = configData["apt-config"] ?? {enableHash: true, label: "apt-stream"};
fixedConfig.repositories = configData.repositories ?? [];
fixedConfig.repositories = (fixedConfig.repositories ?? []).map((repo) => {
if (repo.from === "oci") {
if (!repo.image) throw new Error("oci repository must have image field");
const repoFix: repository = {from: "oci", image: repo.image};
if (repo.platfom_target) repoFix.platfom_target = repo.platfom_target;
if (repo.auth) repoFix.auth = repo.auth;
if (repo["apt-config"]) repoFix["apt-config"] = repo["apt-config"];
if (repo.cronRefresh) repoFix.cronRefresh = repo.cronRefresh;
return repoFix;
} else if (repo.from === "github_release") {
if (!repo.repository) throw new Error("github_release repository must have repository field");
else if (typeof repo.repository !== "string") throw new Error("github_release repository must be string");
const repoFix: repository = {from: "github_release", repository: repo.repository};
if (repo.owner) repoFix.owner = repo.owner;
else {
const [owner, ...repository] = repo.repository.split("/");
if (!owner) throw new Error("github_release repository must have owner field");
if (repository.length === 0) throw new Error("github_release repository must have repository field");
repoFix.owner = owner;
repoFix.repository = repository.join("/");
}
if (repo.token) repoFix.token = repo.token;
if (repo["apt-config"]) repoFix["apt-config"] = repo["apt-config"];
if (repo.cronRefresh) repoFix.cronRefresh = repo.cronRefresh;
return repoFix;
}
return null;
}).filter(a => !!a);
return fixedConfig;
}