rewrite API #3
6
.dockerignore
Normal file
6
.dockerignore
Normal file
@ -0,0 +1,6 @@
|
||||
node_modules/
|
||||
.devcontainer/
|
||||
.vscode/
|
||||
.github/
|
||||
*.d.ts
|
||||
*.js
|
9
Dockerfile
Normal file
9
Dockerfile
Normal file
@ -0,0 +1,9 @@
|
||||
FROM nodejs:latest
|
||||
VOLUME [ "/data" ]
|
||||
WORKDIR /app
|
||||
COPY package*.json ./
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
EXPOSE 3000
|
||||
ENTRYPOINT [ "node", "src/index.js", "--port", "3000", "--config-path", "/data/.apt_stream.yml" ]
|
49
README.md
49
README.md
@ -1,18 +1,52 @@
|
||||
# node-apt
|
||||
# apt-stream
|
||||
|
||||
Crie seu repositorio apt com o nodejs sem precisas salvar arquivos ou mesmo cuidar do armazenamento.
|
||||
Create your apt repository with nodejs without having to save files or even take care of storage.
|
||||
|
||||
## Storages
|
||||
|
||||
Como esse projeto voçe hospedar um repositorio do apt rapida com os seguintes storages:
|
||||
You can host an apt rapida repository with the following storages:
|
||||
|
||||
- Docker and OCI images (find `.deb` files in diff's)
|
||||
- Github Releases
|
||||
- Local folders
|
||||
|
||||
## Config file
|
||||
|
||||
Estou ainda mexendo com o arquivo de configuração dos repositorios, e do servidor por enquanto está com está até eu poder mexer direito nele.
|
||||
|
||||
```yaml
|
||||
# Global apt config
|
||||
apt-config:
|
||||
origin: ""
|
||||
enableHash: true # if it is enabled, it may freeze the request a little because I have to wait for the hashes of the "Packages" files
|
||||
sourcesHost: http://localhost:3000
|
||||
# If you want to use a custom sources.list
|
||||
sourcesList: deb [trusted=yes] %s://%s %s main
|
||||
|
||||
repositories:
|
||||
# Example to docker and OCI image
|
||||
- from: oci
|
||||
image: ghcr.io/sirherobrine23/nodeaptexample
|
||||
# Release endpoint config
|
||||
apt-config:
|
||||
origin: github.com/cli/cli
|
||||
lebel: github-cli
|
||||
description: |
|
||||
This is example
|
||||
is this second line of description.
|
||||
|
||||
# Example to github release
|
||||
- from: github_release
|
||||
repository: cli/cli
|
||||
|
||||
- from: github_release
|
||||
owner: cli
|
||||
repository: cli
|
||||
token: ""
|
||||
```
|
||||
|
||||
## Endpoints
|
||||
|
||||
Esse projeto foi feito com base nas rotas do `archive.ubuntu.com` e do `ftp.debian.org`.
|
||||
This project was made based on the `archive.ubuntu.com` and `ftp.debian.org` routes.
|
||||
|
||||
* `GET` /dists/:package-name/Release
|
||||
* `GET` /dists/:package-name/main/binary-:arch/Packages
|
||||
@ -22,7 +56,4 @@ Esse projeto foi feito com base nas rotas do `archive.ubuntu.com` e do `ftp.debi
|
||||
### Download .deb and package info
|
||||
|
||||
* `GET` /pool/:package_name/:version/:arch.deb - `Download .deb file`
|
||||
* `GET` /pool/:package_name/:version/:arch - `Config package if exists`
|
||||
* `GET` /pool/:package_name/:version - `Get version with config`
|
||||
* `GET` /pool/:package_name - `Get all versions with config`
|
||||
* `GET` /pool - `Get packages registred to packages registry`
|
||||
* `GET` /pool and / - `Get packages registred to packages registry`
|
70
package-lock.json
generated
70
package-lock.json
generated
@ -1,22 +1,27 @@
|
||||
{
|
||||
"name": "node-apt",
|
||||
"name": "apt-stream",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "node-apt",
|
||||
"name": "apt-stream",
|
||||
"version": "1.0.0",
|
||||
"license": "GPL-2.0",
|
||||
"dependencies": {
|
||||
"@sirherobrine23/coreutils": "^2.1.6",
|
||||
"cron": "^2.1.0",
|
||||
"express": "^4.18.2",
|
||||
"lzma-native": "^8.0.6",
|
||||
"tar": "^6.1.13",
|
||||
"yaml": "^2.1.3",
|
||||
"yargs": "^17.6.2"
|
||||
},
|
||||
"bin": {
|
||||
"apt-stream": "src/index.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cron": "^2.0.0",
|
||||
"@types/express": "^4.17.15",
|
||||
"@types/lzma-native": "^4.0.1",
|
||||
"@types/node": "^18.11.17",
|
||||
@ -272,6 +277,16 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/cron": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz",
|
||||
"integrity": "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/luxon": "*",
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/express": {
|
||||
"version": "4.17.15",
|
||||
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.15.tgz",
|
||||
@ -300,6 +315,12 @@
|
||||
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz",
|
||||
"integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ=="
|
||||
},
|
||||
"node_modules/@types/luxon": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.1.0.tgz",
|
||||
"integrity": "sha512-gCd/HcCgjqSxfMrgtqxCgYk/22NBQfypwFUG7ZAyG/4pqs51WLTcUzVp1hqTbieDYeHS3WoVEh2Yv/2l+7B0Vg==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/lzma-native": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/lzma-native/-/lzma-native-4.0.1.tgz",
|
||||
@ -642,6 +663,14 @@
|
||||
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/cron": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz",
|
||||
"integrity": "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w==",
|
||||
"dependencies": {
|
||||
"luxon": "^1.23.x"
|
||||
}
|
||||
},
|
||||
"node_modules/cssom": {
|
||||
"version": "0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
|
||||
@ -1297,6 +1326,14 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/luxon": {
|
||||
"version": "1.28.0",
|
||||
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
|
||||
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==",
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/lzma-native": {
|
||||
"version": "8.0.6",
|
||||
"resolved": "https://registry.npmjs.org/lzma-native/-/lzma-native-8.0.6.tgz",
|
||||
@ -2468,6 +2505,16 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/cron": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz",
|
||||
"integrity": "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/luxon": "*",
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/express": {
|
||||
"version": "4.17.15",
|
||||
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.15.tgz",
|
||||
@ -2496,6 +2543,12 @@
|
||||
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz",
|
||||
"integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ=="
|
||||
},
|
||||
"@types/luxon": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.1.0.tgz",
|
||||
"integrity": "sha512-gCd/HcCgjqSxfMrgtqxCgYk/22NBQfypwFUG7ZAyG/4pqs51WLTcUzVp1hqTbieDYeHS3WoVEh2Yv/2l+7B0Vg==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/lzma-native": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/lzma-native/-/lzma-native-4.0.1.tgz",
|
||||
@ -2776,6 +2829,14 @@
|
||||
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
|
||||
"dev": true
|
||||
},
|
||||
"cron": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz",
|
||||
"integrity": "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w==",
|
||||
"requires": {
|
||||
"luxon": "^1.23.x"
|
||||
}
|
||||
},
|
||||
"cssom": {
|
||||
"version": "0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
|
||||
@ -3262,6 +3323,11 @@
|
||||
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
|
||||
"integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ=="
|
||||
},
|
||||
"luxon": {
|
||||
"version": "1.28.0",
|
||||
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
|
||||
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
|
||||
},
|
||||
"lzma-native": {
|
||||
"version": "8.0.6",
|
||||
"resolved": "https://registry.npmjs.org/lzma-native/-/lzma-native-8.0.6.tgz",
|
||||
|
16
package.json
16
package.json
@ -1,21 +1,21 @@
|
||||
{
|
||||
"name": "node-apt",
|
||||
"name": "apt-stream",
|
||||
"version": "1.0.0",
|
||||
"description": "Replace",
|
||||
"main": "src/index.js",
|
||||
"types": "./src/index.d.ts",
|
||||
"private": false,
|
||||
"type": "module",
|
||||
"homepage": "https://github.com/Sirherobrine23/node-apt#readme",
|
||||
"homepage": "https://github.com/Sirherobrine23/apt-stream#readme",
|
||||
"author": "Matheus Sampaio Queiroga <srherobrine20@gmail.com> (https://sirherobrine23.org/)",
|
||||
"license": "GPL-2.0",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/Sirherobrine23/node-apt.git"
|
||||
"url": "git+https://github.com/Sirherobrine23/apt-stream.git"
|
||||
},
|
||||
"keywords": [],
|
||||
"bugs": {
|
||||
"url": "https://github.com/Sirherobrine23/node-apt/issues"
|
||||
"url": "https://github.com/Sirherobrine23/apt-stream/issues"
|
||||
},
|
||||
"sponsor": {
|
||||
"url": "https://github.com/sponsors/Sirherobrine23"
|
||||
@ -26,10 +26,15 @@
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"apt-stream": "./src/index.js"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "ts-node src/index.ts"
|
||||
"start": "ts-node src/index.ts",
|
||||
"build": "tsc"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cron": "^2.0.0",
|
||||
"@types/express": "^4.17.15",
|
||||
"@types/lzma-native": "^4.0.1",
|
||||
"@types/node": "^18.11.17",
|
||||
@ -40,6 +45,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@sirherobrine23/coreutils": "^2.1.6",
|
||||
"cron": "^2.1.0",
|
||||
"express": "^4.18.2",
|
||||
"lzma-native": "^8.0.6",
|
||||
"tar": "^6.1.13",
|
||||
|
@ -1,13 +1,22 @@
|
||||
version: 1
|
||||
repos: []
|
||||
# - repo: ghcr.io/sirherobrine23/nodeaptexample:sha256:c7b3eb9435f67aa08a2e98fdaa5c4548bb8e410fc804ed352bb03dcb212dde96
|
||||
# from: oci
|
||||
# Global apt config
|
||||
apt-config:
|
||||
origin: ""
|
||||
enableHash: true # if it is enabled, it may freeze the request a little because I have to wait for the hashes of the "Packages" files
|
||||
sourcesHost: http://localhost:3000
|
||||
# If you want to use a custom sources.list
|
||||
sourcesList: deb [trusted=yes] %s://%s %s main
|
||||
|
||||
# - repo: Sirherobrine23/ghcrAptRepository
|
||||
# from: release
|
||||
repositories:
|
||||
# Example to docker and OCI image
|
||||
# - from: oci
|
||||
# image: ghcr.io/sirherobrine23/nodeaptexample
|
||||
# # Release endpoint config
|
||||
# apt-config:
|
||||
# origin: github.com/cli/cli
|
||||
# lebel: github-cli
|
||||
|
||||
# - repo: The-Bds-Maneger/BedrockFetch
|
||||
# from: release
|
||||
|
||||
# - repo: cli/cli
|
||||
# from: release
|
||||
# Example to github release
|
||||
- from: github_release
|
||||
repository: cli
|
||||
owner: cli
|
||||
takeUpTo: 2
|
335
src/aptRepo.ts
335
src/aptRepo.ts
@ -1,335 +0,0 @@
|
||||
import { format } from "node:util";
|
||||
import { WriteStream } from "node:fs";
|
||||
import { PassThrough, Readable, Writable } from "node:stream";
|
||||
import { getConfig } from "./repoConfig.js";
|
||||
import * as ghcr from "./oci_registry.js";
|
||||
import * as release from "./githubRelease.js";
|
||||
import zlib from "node:zlib";
|
||||
import express from "express";
|
||||
import coreUtils from "@sirherobrine23/coreutils";
|
||||
|
||||
export type packagesObject = {
|
||||
Package: string
|
||||
Version: string,
|
||||
/** endpoint folder file */
|
||||
Filename: string,
|
||||
"Installed-Size": number,
|
||||
Maintainer: string,
|
||||
Architecture: string,
|
||||
Depends?: string,
|
||||
Homepage?: string,
|
||||
Section?: string,
|
||||
Priority?: string,
|
||||
Size: number,
|
||||
MD5sum: string,
|
||||
SHA256: string,
|
||||
Description?: string,
|
||||
};
|
||||
|
||||
export type ReleaseOptions = {
|
||||
Origin?: string,
|
||||
Suite?: string,
|
||||
Archive?: string,
|
||||
lebel?: string,
|
||||
Codename?: string,
|
||||
Architectures: string[],
|
||||
Components: string[],
|
||||
Description?: string,
|
||||
sha256?: {sha256: string, size: number, file: string}[]
|
||||
};
|
||||
|
||||
export function parseDebControl(control: string|Buffer) {
|
||||
if (Buffer.isBuffer(control)) control = control.toString();
|
||||
const controlObject: {[key: string]: string} = {};
|
||||
for (const line of control.split(/\r?\n/)) {
|
||||
if (/^[\w\S]+:/.test(line)) {
|
||||
const [, key, value] = line.match(/^([\w\S]+):(.*)$/);
|
||||
controlObject[key.trim()] = value.trim();
|
||||
} else {
|
||||
controlObject[Object.keys(controlObject).at(-1)] += line;
|
||||
}
|
||||
}
|
||||
return controlObject;
|
||||
}
|
||||
|
||||
export function mountRelease(repo: ReleaseOptions) {
|
||||
let data = [`Lebel: ${repo.lebel||repo.Origin}`];
|
||||
data.push(`Date: ${(new Date()).toUTCString()}`);
|
||||
if (repo.Origin) data.push(`Origin: ${repo.Origin}`);
|
||||
if (repo.Suite) data.push(`Suite: ${repo.Suite}`);
|
||||
else if (repo.Archive) data.push(`Archive: ${repo.Archive}`);
|
||||
if (repo.Codename) data.push(`Codename: ${repo.Codename}`);
|
||||
data.push(`Architectures: ${repo.Architectures.join(" ")}\nComponents: ${repo.Components.join(" ")}`);
|
||||
if (repo.Description) data.push(`Description: ${repo.Description}`);
|
||||
if (repo.sha256 && repo.sha256?.length > 0) {
|
||||
data.push("SHA256:");
|
||||
for (const file of repo.sha256) {
|
||||
data.push(` ${file.sha256} ${file.size} ${file.file}`);
|
||||
}
|
||||
}
|
||||
return data.join("\n")+"\n";
|
||||
}
|
||||
|
||||
export type registryPackageData = {
|
||||
name: string,
|
||||
getStrem: () => Promise<Readable>,
|
||||
version: string,
|
||||
arch: string,
|
||||
size?: number,
|
||||
from?: string,
|
||||
signature?: {
|
||||
sha256: string,
|
||||
md5: string,
|
||||
},
|
||||
packageConfig?: {
|
||||
[key: string]: string;
|
||||
}
|
||||
};
|
||||
|
||||
type localRegister = {
|
||||
[name: string]: {
|
||||
[version: string]: {
|
||||
[arch: string]: {
|
||||
getStream: registryPackageData["getStrem"],
|
||||
config?: registryPackageData["packageConfig"],
|
||||
signature?: registryPackageData["signature"],
|
||||
size?: registryPackageData["size"],
|
||||
from?: registryPackageData["from"],
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export class localRegistryManeger {
|
||||
public packageRegister: localRegister = {};
|
||||
prettyPackages() {
|
||||
const packagePretty = {};
|
||||
for (const name in this.packageRegister) {
|
||||
if (!packagePretty[name]) packagePretty[name] = [];
|
||||
for (const version in this.packageRegister[name]) {
|
||||
for (const arch in this.packageRegister[name][version]) {
|
||||
packagePretty[name].push({
|
||||
version,
|
||||
arch,
|
||||
config: this.packageRegister[name][version][arch].config,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return packagePretty;
|
||||
}
|
||||
|
||||
public registerPackage(packageConfig: registryPackageData) {
|
||||
packageConfig.name = packageConfig.name?.toLowerCase()?.trim();
|
||||
packageConfig.version = packageConfig?.version?.trim();
|
||||
if (!this.packageRegister) this.packageRegister = {};
|
||||
if (!this.packageRegister[packageConfig.name]) this.packageRegister[packageConfig.name] = {};
|
||||
if (!this.packageRegister[packageConfig.name][packageConfig.version]) this.packageRegister[packageConfig.name][packageConfig.version] = {};
|
||||
console.log("[Internal package maneger]: Registry %s with version %s and arch %s", packageConfig.name, packageConfig.version, packageConfig.arch);
|
||||
this.packageRegister[packageConfig.name][packageConfig.version][packageConfig.arch] = {
|
||||
getStream: packageConfig.getStrem,
|
||||
config: packageConfig.packageConfig,
|
||||
signature: packageConfig.signature,
|
||||
size: packageConfig.size,
|
||||
};
|
||||
}
|
||||
|
||||
public async createPackage(compress: boolean, name: string, arch: string, res: WriteStream|Writable|PassThrough = new PassThrough({read(){}, write(){}})) {
|
||||
const sign: {sha256?: {hash: string, size: number}, md5?: {hash: string, size: number}, size: number} = {size: 0};
|
||||
const Packages: packagesObject[] = [];
|
||||
if (!this.packageRegister[name]) throw new Error(format("%s not found", name));
|
||||
for (const version in this.packageRegister[name]) {
|
||||
const packageData = this.packageRegister[name][version][arch];
|
||||
if (!packageData) continue;
|
||||
Packages.push({
|
||||
Package: name,
|
||||
Version: version,
|
||||
Filename: format("pool/%s/%s/%s.deb", name, version, arch),
|
||||
"Installed-Size": parseInt(packageData.config?.["Installed-Size"]||"0"),
|
||||
Maintainer: packageData.config?.Maintainer||"node-apt",
|
||||
Architecture: packageData.config?.Architecture||"all",
|
||||
Depends: packageData.config?.Depends,
|
||||
Homepage: packageData.config?.Homepage,
|
||||
Section: packageData.config?.Section,
|
||||
Priority: packageData.config?.Priority,
|
||||
Size: packageData.size||0,
|
||||
MD5sum: packageData.signature?.md5,
|
||||
SHA256: packageData.signature?.sha256,
|
||||
Description: packageData.config?.Description,
|
||||
});
|
||||
}
|
||||
let vsize = 0;
|
||||
const rawStream = new PassThrough({read(){}, write(){}});
|
||||
if (compress) {
|
||||
const ReadStream = new PassThrough({read(){}, write(){}});
|
||||
const gzip = ReadStream.pipe(zlib.createGzip());
|
||||
gzip.pipe(res);
|
||||
gzip.pipe(rawStream);
|
||||
gzip.on("data", (chunk) => vsize += chunk.length);
|
||||
res = ReadStream;
|
||||
} else rawStream.on("data", (chunk) => vsize += chunk.length);
|
||||
let waitHashs = coreUtils.extendsCrypto.createSHA256_MD5(rawStream, "both", new Promise(done => {
|
||||
rawStream.on("end", () => setTimeout(done, 500));
|
||||
rawStream.on("close", () => setTimeout(done, 500));
|
||||
})).then(data => {
|
||||
sign.sha256 = {hash: data.sha256, size: vsize};
|
||||
sign.md5 = {hash: data.md5, size: vsize};
|
||||
});
|
||||
for (const packageInfo of Packages) {
|
||||
let packageData = [];
|
||||
for (let i in packageInfo) packageData.push(`${i}: ${packageInfo[i]||""}`);
|
||||
const configLine = packageData.join("\n")+"\n\n";
|
||||
sign.size += configLine.length;
|
||||
rawStream.push(Buffer.from(configLine, "utf8"));
|
||||
rawStream.write(Buffer.from(configLine, "utf8"));
|
||||
if (res instanceof PassThrough) res.push(configLine);
|
||||
else res.write(configLine);
|
||||
}
|
||||
res.end();
|
||||
res.destroy();
|
||||
rawStream.end();
|
||||
rawStream.destroy();
|
||||
await waitHashs;
|
||||
return sign;
|
||||
}
|
||||
}
|
||||
|
||||
async function mainConfig(configPath: string) {
|
||||
const config = await getConfig(configPath);
|
||||
const packageReg = new localRegistryManeger();
|
||||
Promise.all(config.repos.map(async repo => {
|
||||
if (repo.from === "release") return release.fullConfig({config: repo.repo, githubToken: repo?.auth?.password}, () => {}).catch(console.error);
|
||||
if (repo.from === "oci") return ghcr.fullConfig({image: repo.repo, targetInfo: repo.ociConfig}, () => {}).catch(console.error);
|
||||
}));
|
||||
return packageReg;
|
||||
}
|
||||
|
||||
export type apiConfig = {
|
||||
configPath: string,
|
||||
portListen?: number,
|
||||
callback?: (port: number) => void,
|
||||
repositoryOptions?: {
|
||||
Origin?: string,
|
||||
lebel?: string,
|
||||
}
|
||||
};
|
||||
export async function createAPI(apiConfig: apiConfig) {
|
||||
const mainRegister = await mainConfig(apiConfig.configPath);
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use(express.urlencoded({extended: true}));
|
||||
app.use((_req, res, next) => {
|
||||
res.json = (data) => res.setHeader("Content-Type", "application/json").send(JSON.stringify(data, null, 2));
|
||||
next();
|
||||
});
|
||||
|
||||
// Request log
|
||||
app.use((req, _res, next) => {
|
||||
next();
|
||||
console.log("[%s]: From: %s, path: %s", req.protocol, req.ip, req.path);
|
||||
});
|
||||
|
||||
// /dists
|
||||
// Signed Release with gpg
|
||||
// app.get("/dists/:suite/InRelease", (req, res) => {});
|
||||
// app.get("/dists/:package/Release.gpg", (req, res) => {});
|
||||
|
||||
// Root release
|
||||
app.get("/dists/:package/Release", async (req, res) => {
|
||||
if (!mainRegister.packageRegister[req.params.package]) return res.status(404).json({error: "Package not registred"});
|
||||
const Archs: string[] = [];
|
||||
Object.keys(mainRegister.packageRegister[req.params.package]).forEach(version => Object.keys(mainRegister.packageRegister[req.params.package][version]).forEach(arch => (!Archs.includes(arch.toLowerCase()))?Archs.push(arch.toLowerCase()):null));
|
||||
const shas: ReleaseOptions["sha256"] = [];
|
||||
for (const arch of Archs) {
|
||||
const Packagegz = await mainRegister.createPackage(true, req.params.package, arch);
|
||||
const Package = await mainRegister.createPackage(false, req.params.package, arch);
|
||||
if (Packagegz?.sha256?.hash) shas.push({file: format("main/binary-%s/Packages.gz", arch), sha256: Packagegz.sha256.hash, size: Packagegz.sha256.size});
|
||||
if (Package?.sha256?.hash) shas.push({file: format("main/binary-%s/Packages", arch), sha256: Package.sha256.hash, size: Package.sha256.size});
|
||||
}
|
||||
const data = mountRelease({
|
||||
Origin: apiConfig.repositoryOptions?.Origin||"node-apt",
|
||||
lebel: apiConfig.repositoryOptions?.lebel,
|
||||
Suite: req.params.package,
|
||||
Components: ["main"],
|
||||
Architectures: Archs,
|
||||
sha256: shas
|
||||
});
|
||||
res.setHeader("Content-Type", "text/plain");
|
||||
res.setHeader("Content-Length", data.length);
|
||||
return res.send(data);
|
||||
});
|
||||
|
||||
// Binary release
|
||||
app.get("/dists/:package/main/binary-:arch/Release", (req, res) => {
|
||||
const Archs: string[] = [];
|
||||
Object.keys(mainRegister.packageRegister[req.params.package]).forEach(version => Object.keys(mainRegister.packageRegister[req.params.package][version]).forEach(arch => (!Archs.includes(arch.toLowerCase()))?Archs.push(arch.toLowerCase()):null));
|
||||
if (!Archs.includes(req.params.arch.toLowerCase())) return res.status(404).json({error: "Package arch registred"});
|
||||
res.setHeader("Content-Type", "text/plain");
|
||||
return res.send(mountRelease({
|
||||
Origin: apiConfig.repositoryOptions?.Origin||"node-apt",
|
||||
lebel: apiConfig.repositoryOptions?.lebel,
|
||||
Archive: req.params.package,
|
||||
Components: ["main"],
|
||||
Architectures: [req.params.arch.toLowerCase()]
|
||||
}));
|
||||
});
|
||||
|
||||
// binary packages
|
||||
app.get("/dists/:package/main/binary-:arch/Packages(.gz|)", (req, res, next) => mainRegister.createPackage(req.path.endsWith(".gz"), req.params.package, req.params.arch, res.writeHead(200, {"Content-Type": req.path.endsWith(".gz") ? "application/x-gzip" : "text/plain"})).catch(next));
|
||||
|
||||
// source.list
|
||||
app.get("/*.list", (req, res) => {
|
||||
res.setHeader("Content-type", "text/plain");
|
||||
let config = "";
|
||||
Object.keys(mainRegister.packageRegister).forEach(packageName => config += format("deb [trusted=yes] %s://%s %s main\n", req.protocol, req.headers.host, packageName));
|
||||
res.send(config+"\n");
|
||||
});
|
||||
|
||||
// Pool
|
||||
app.get(["/", "/pool"], (_req, res) => res.json(mainRegister.prettyPackages()));
|
||||
app.get("/pool/:package_name", (req, res) => {
|
||||
const {package_name} = req.params;
|
||||
const info = mainRegister.packageRegister[package_name];
|
||||
if (!info) return res.status(404).json({error: "Package not registred"});
|
||||
return res.json(info);
|
||||
});
|
||||
app.get("/pool/:package_name/:version", (req, res) => {
|
||||
const {package_name, version} = req.params;
|
||||
const info = mainRegister.packageRegister[package_name];
|
||||
if (!info) return res.status(404).json({error: "Package not registred"});
|
||||
const ver = info?.[version];
|
||||
if (!ver) return res.status(404).json({error: "version not registred"});
|
||||
return res.json(ver);
|
||||
});
|
||||
|
||||
app.get("/pool/:package_name/:version/:arch.deb", (req, res) => {
|
||||
const {package_name, arch, version} = req.params;
|
||||
const info = mainRegister.packageRegister[package_name];
|
||||
if (!info) return res.status(404).json({error: "Package not registred"});
|
||||
const ver = info?.[version];
|
||||
if (!ver) return res.status(404).json({error: "version not registred"});
|
||||
const archInfo = ver?.[arch];
|
||||
if (!archInfo) return res.status(404).json({error: "arch not registred"});
|
||||
const stream = archInfo?.getStream;
|
||||
if (!stream) return res.status(404).json({error: "Package not registred"});
|
||||
res.writeHead(200, {"Content-Type": "application/x-debian-package"});
|
||||
return Promise.resolve(stream()).then(stream => stream.pipe(res));
|
||||
});
|
||||
|
||||
app.get("/pool/:package_name/:version/:arch", (req, res) => {
|
||||
const {package_name, arch, version} = req.params;
|
||||
const info = mainRegister.packageRegister[package_name];
|
||||
if (!info) return res.status(404).json({error: "Package not registred"});
|
||||
const ver = info?.[version];
|
||||
if (!ver) return res.status(404).json({error: "version not registred"});
|
||||
const archInfo = ver?.[arch];
|
||||
if (!archInfo) return res.status(404).json({error: "arch not registred"});
|
||||
return res.json(archInfo);
|
||||
});
|
||||
|
||||
app.listen(apiConfig.portListen||0, function listen() {
|
||||
if (typeof apiConfig.callback !== "function") apiConfig.callback = (port) => console.log("API listen on port %s", port);
|
||||
apiConfig.callback(this.address().port);
|
||||
});
|
||||
return app;
|
||||
}
|
@ -19,7 +19,7 @@ export function packageManeger(RootOptions?: backendConfig) {
|
||||
const localRegister: registerOobject = {};
|
||||
function pushPackage(control: packageControl, getStream: () => Promise<Readable>, from?: string) {
|
||||
if (!localRegister[control.Package]) localRegister[control.Package] = [];
|
||||
console.log("Register %s/%s-5s", control.Package, control.Version, control.Architecture);
|
||||
// console.log("Register %s/%s-5s", control.Package, control.Version, control.Architecture);
|
||||
localRegister[control.Package].push({
|
||||
getStream,
|
||||
control,
|
||||
@ -124,9 +124,9 @@ export function packageManeger(RootOptions?: backendConfig) {
|
||||
|
||||
async function createRelease(options?: {packageName?: string, Arch?: string, includesHashs?: boolean, component?: string[], archive?: string}) {
|
||||
const textLines = [];
|
||||
if (RootOptions?.aptConfig?.origin) textLines.push(`Origin: ${RootOptions.aptConfig.origin}`);
|
||||
if (RootOptions?.["apt-config"]?.origin) textLines.push(`Origin: ${RootOptions["apt-config"].origin}`);
|
||||
if (options?.archive) textLines.push(`Archive: ${options?.archive}`);
|
||||
else textLines.push(`Lebel: ${RootOptions?.aptConfig?.label||"node-apt"}`, `Date: ${new Date().toUTCString()}`);
|
||||
else textLines.push(`Lebel: ${RootOptions?.["apt-config"]?.label||"node-apt"}`, `Date: ${new Date().toUTCString()}`);
|
||||
const components = options?.component ?? ["main"];
|
||||
const archs: string[] = [];
|
||||
|
||||
@ -190,7 +190,7 @@ export default async function repo(aptConfig?: backendConfig) {
|
||||
const app = express();
|
||||
const registry = packageManeger();
|
||||
app.disable("x-powered-by").disable("etag").use(express.json()).use(express.urlencoded({extended: true})).use((_req, res, next) => {
|
||||
res.json = (data) => res.setHeader("Content-Type", "application/json").send(JSON.stringify(data, null, 2));
|
||||
res.json = (data) => res.setHeader("Content-Type", "application/json").send(JSON.stringify(data ?? null, null, 2));
|
||||
next();
|
||||
}).use((req, _res, next) => {
|
||||
next();
|
||||
@ -223,7 +223,7 @@ export default async function repo(aptConfig?: backendConfig) {
|
||||
if (!Targets) Targets = ["all"];
|
||||
res.setHeader("Content-type", "text/plain");
|
||||
let config = "";
|
||||
for (const target of Targets) config += format(aptConfig?.aptConfig?.sourcesList ?? "deb [trusted=yes] %s://%s %s main\n", req.protocol, host, target);
|
||||
for (const target of Targets) config += format(aptConfig?.["apt-config"]?.sourcesList ?? "deb [trusted=yes] %s://%s %s main\n", req.protocol, host, target);
|
||||
res.send(config+"\n");
|
||||
});
|
||||
|
||||
@ -244,7 +244,7 @@ export default async function repo(aptConfig?: backendConfig) {
|
||||
const { suite, arch } = req.params;
|
||||
res.setHeader("Content-Type", "text/plain");
|
||||
return registry.createRelease({
|
||||
includesHashs: aptConfig?.aptConfig?.enableHash ?? true,
|
||||
includesHashs: aptConfig?.["apt-config"]?.enableHash ?? true,
|
||||
Arch: arch,
|
||||
packageName: suite === "all" ? undefined : suite,
|
||||
component: [req.params.component],
|
||||
|
@ -46,7 +46,7 @@ export function createExtract(fn: (info: fileInfo, stream: Readable) => void) {
|
||||
}
|
||||
}
|
||||
size -= chunk.length;
|
||||
entryStream.push(chunk, "binary");
|
||||
if (entryStream) entryStream.push(chunk, "binary");
|
||||
return callback();
|
||||
}
|
||||
let waitMore: Buffer;
|
||||
@ -64,7 +64,7 @@ export function createExtract(fn: (info: fileInfo, stream: Readable) => void) {
|
||||
callback();
|
||||
}
|
||||
if (!chunk.subarray(0, 8).toString().trim().startsWith("!<arch>")) {
|
||||
this.destroy();
|
||||
__writed.destroy();
|
||||
return callback(new Error("Not an ar file"));
|
||||
}
|
||||
__locked = true;
|
||||
|
@ -39,7 +39,7 @@ export function parseControl(rawControlFile: string) {
|
||||
|
||||
export type debReturn = Awaited<ReturnType<typeof extractDebControl>>;
|
||||
|
||||
export async function extractDebControl(debStream: Readable) {
|
||||
export async function extractDebControl(debStream: Readable, endPromise: Promise<void> = new Promise(done => debStream.once("end", done))) {
|
||||
return new Promise<{size: number, control: packageControl}>((done, reject) => {
|
||||
let fileSize = 0;
|
||||
debStream.on("data", (chunk) => fileSize += chunk.length);
|
||||
@ -53,7 +53,7 @@ export async function extractDebControl(debStream: Readable) {
|
||||
controlEntry.on("data", chunck => controlFile = (!controlFile)?chunck:Buffer.concat([controlFile, chunck])).once("end", async () => {
|
||||
const sign = await signs;
|
||||
const control = parseControl(controlFile.toString());
|
||||
debStream.on("end", () => {
|
||||
endPromise.then(() => {
|
||||
control.MD5sum = sign.md5;
|
||||
control.SHA256 = sign.sha256;
|
||||
control.Size = fileSize;
|
||||
|
@ -18,14 +18,14 @@ export async function fullConfig(config: {config: string|baseOptions<{releaseTag
|
||||
};
|
||||
}).filter(({assets}) => assets?.length > 0);
|
||||
|
||||
for (const rel of releases) {
|
||||
for (const asset of rel.assets) {
|
||||
const getStream = async () => coreUtils.httpRequest.pipeFetch(asset.download)
|
||||
const control = await extractDebControl(await getStream());
|
||||
fn({
|
||||
...control,
|
||||
getStream,
|
||||
});
|
||||
}
|
||||
}
|
||||
return Promise.all(releases.map(async (rel) => Promise.all(rel.assets.map(async (asset) => {
|
||||
console.log(`Downloading ${asset.name} from ${rel.tag}`);
|
||||
const getStream = async () => coreUtils.httpRequest.pipeFetch(asset.download)
|
||||
const control = await getStream().then(Stream => extractDebControl(Stream, new Promise(done => Stream.once("end", done))));
|
||||
fn({
|
||||
...control,
|
||||
getStream,
|
||||
});
|
||||
return {asset, getStream, control};
|
||||
})))).then(data => data.flat());
|
||||
}
|
||||
|
39
src/index.ts
39
src/index.ts
@ -4,6 +4,7 @@ import repo from "./apt_repo_v2.js";
|
||||
import { getConfig } from "./repoConfig.js";
|
||||
import github_release from "./githubRelease.js";
|
||||
import oci_registry from "./oci_registry.js";
|
||||
import { CronJob } from "cron";
|
||||
|
||||
yargs(process.argv.slice(2)).wrap(null).strict().help().option("cofig-path", {
|
||||
type: "string",
|
||||
@ -12,12 +13,34 @@ yargs(process.argv.slice(2)).wrap(null).strict().help().option("cofig-path", {
|
||||
type: "number",
|
||||
default: 3000,
|
||||
}).parseAsync().then(async options => {
|
||||
const { app, registry } = await repo();
|
||||
app.listen(options.port, () => {
|
||||
console.log(`Server listening on port ${options.port}`);
|
||||
});
|
||||
Promise.all((await getConfig(options["cofig-path"])).repos.map(async repo => {
|
||||
if (repo.from === "oci") return oci_registry({image: repo.repo, targetInfo: repo.ociConfig}, data => registry.pushPackage(data.control, data.getStream)).catch(console.error);
|
||||
else if (repo.from === "release") return github_release({config: repo.repo, githubToken: repo.auth.password}, data => registry.pushPackage(data.control, data.getStream)).catch(console.error);
|
||||
})).catch(console.error);
|
||||
const config = await getConfig(options["cofig-path"]);
|
||||
const { app, registry } = await repo(config);
|
||||
app.listen(options.port, () => console.log(`Server listening on port ${options.port}`));
|
||||
for (const repo of config.repositories) {
|
||||
console.log(repo);
|
||||
const sen = () => Promise.resolve().then(async () => {
|
||||
if (repo.from === "github_release") {
|
||||
const tags = repo.tags ?? ["latest"];
|
||||
if (tags.length === 0) tags.push("latest");
|
||||
return Promise.all(tags.map(async tag => github_release({
|
||||
githubToken: repo.token,
|
||||
config: {
|
||||
owner: repo.owner,
|
||||
repo: repo.repository,
|
||||
releaseTag: tag,
|
||||
}
|
||||
}, ({control, getStream}) => registry.pushPackage(control, getStream))));
|
||||
} else if (repo.from === "oci") {
|
||||
return oci_registry({
|
||||
image: repo.image,
|
||||
targetInfo: repo.platfom_target,
|
||||
}, ({control, getStream}) => registry.pushPackage(control, getStream));
|
||||
}
|
||||
return null;
|
||||
}).catch(console.trace);
|
||||
sen();
|
||||
const cronJobs = repo.cronRefresh ?? [];
|
||||
const jobs = cronJobs.map(cron => new CronJob(cron, sen));
|
||||
jobs.forEach(job => job.start());
|
||||
}
|
||||
});
|
@ -18,11 +18,16 @@ export async function fullConfig(imageInfo: {image: string, targetInfo?: DockerR
|
||||
return fn({
|
||||
...control,
|
||||
getStream: async () => {
|
||||
return new Promise<Readable>((done, reject) => registry.blobLayerStream(data.layer.digest).then(stream => stream.pipe(tar.list({
|
||||
onentry(getEntry) {
|
||||
if (getEntry.path === entry.path) return done(getEntry as any);
|
||||
}
|
||||
}))).catch(reject));
|
||||
return new Promise<Readable>((done, reject) => registry.blobLayerStream(data.layer.digest).then(stream => {
|
||||
stream.on("error", reject);
|
||||
stream.pipe(tar.list({
|
||||
onentry(getEntry) {
|
||||
if (getEntry.path !== entry.path) return null;
|
||||
return done(getEntry as any);
|
||||
}
|
||||
// @ts-ignore
|
||||
}).on("error", reject));
|
||||
}).catch(reject));
|
||||
},
|
||||
});
|
||||
},
|
||||
|
@ -2,58 +2,74 @@ import coreUtils, { DockerRegistry } from "@sirherobrine23/coreutils";
|
||||
import * as yaml from "yaml";
|
||||
import fs from "node:fs/promises";
|
||||
|
||||
export type configV1 = {
|
||||
version: 1,
|
||||
repos: (({
|
||||
from: "release",
|
||||
repo: string|{
|
||||
owner: string,
|
||||
repo: string
|
||||
},
|
||||
}|{
|
||||
from: "oci",
|
||||
repo: string,
|
||||
ociConfig?: DockerRegistry.Manifest.platfomTarget,
|
||||
}) & {
|
||||
auth?: {
|
||||
username?: string,
|
||||
password?: string
|
||||
}
|
||||
})[]
|
||||
export type apt_config = {
|
||||
origin?: string,
|
||||
label?: string,
|
||||
enableHash?: boolean,
|
||||
sourcesList?: string
|
||||
};
|
||||
|
||||
export type backendConfig = {
|
||||
aptConfig?: {
|
||||
origin?: string,
|
||||
label?: string,
|
||||
enableHash?: boolean,
|
||||
sourcesList?: string
|
||||
export type repository = ({
|
||||
from: "oci",
|
||||
image: string,
|
||||
platfom_target?: DockerRegistry.Manifest.platfomTarget
|
||||
auth?: {
|
||||
username?: string,
|
||||
password?: string
|
||||
},
|
||||
repositorys: {
|
||||
target: "oci_registry"|"github_release",
|
||||
repo: string,
|
||||
}[]
|
||||
};
|
||||
|
||||
export async function getConfig(filePath: string): Promise<configV1> {
|
||||
if (!await coreUtils.extendFs.exists(filePath)) throw new Error("file not exists");
|
||||
const configData: configV1 = yaml.parse(await fs.readFile(filePath, "utf8"));
|
||||
return {
|
||||
version: 1,
|
||||
repos: (configData?.repos ?? []).map(data => {
|
||||
if (data.from === "oci" && typeof data.repo === "string") {
|
||||
return {
|
||||
repo: data.repo,
|
||||
from: "oci",
|
||||
auth: data.auth,
|
||||
ociConfig: data.ociConfig
|
||||
};
|
||||
}
|
||||
return {
|
||||
repo: (typeof data.repo === "string")?data.repo:{owner: data.repo.owner, repo: data.repo.repo},
|
||||
from: "release",
|
||||
auth: data.auth,
|
||||
}
|
||||
})
|
||||
};
|
||||
"apt-config"?: apt_config,
|
||||
}|{
|
||||
from: "github_release",
|
||||
repository: string,
|
||||
owner?: string,
|
||||
tags?: string[],
|
||||
takeUpTo?: number,
|
||||
token?: string,
|
||||
"apt-config"?: apt_config,
|
||||
}) & {
|
||||
/** cron range: https://github.com/kelektiv/node-cron#cron-ranges */
|
||||
cronRefresh?: string[],
|
||||
}
|
||||
|
||||
export type backendConfig = Partial<{
|
||||
"apt-config"?: apt_config,
|
||||
repositories: repository[]
|
||||
}>;
|
||||
|
||||
export async function getConfig(filePath: string) {
|
||||
if (!await coreUtils.extendFs.exists(filePath)) throw new Error("config File not exists");
|
||||
const fixedConfig: backendConfig = {};
|
||||
const configData: backendConfig = yaml.parse(await fs.readFile(filePath, "utf8"));
|
||||
fixedConfig["apt-config"] = configData["apt-config"] ?? {enableHash: true, label: "apt-stream"};
|
||||
fixedConfig.repositories = configData.repositories ?? [];
|
||||
fixedConfig.repositories = (fixedConfig.repositories ?? []).map((repo) => {
|
||||
if (repo.from === "oci") {
|
||||
if (!repo.image) throw new Error("oci repository must have image field");
|
||||
const repoFix: repository = {from: "oci", image: repo.image};
|
||||
if (repo.platfom_target) repoFix.platfom_target = repo.platfom_target;
|
||||
if (repo.auth) repoFix.auth = repo.auth;
|
||||
if (repo["apt-config"]) repoFix["apt-config"] = repo["apt-config"];
|
||||
if (repo.cronRefresh) repoFix.cronRefresh = repo.cronRefresh;
|
||||
return repoFix;
|
||||
} else if (repo.from === "github_release") {
|
||||
if (!repo.repository) throw new Error("github_release repository must have repository field");
|
||||
else if (typeof repo.repository !== "string") throw new Error("github_release repository must be string");
|
||||
const repoFix: repository = {from: "github_release", repository: repo.repository};
|
||||
if (repo.owner) repoFix.owner = repo.owner;
|
||||
else {
|
||||
const [owner, ...repository] = repo.repository.split("/");
|
||||
if (!owner) throw new Error("github_release repository must have owner field");
|
||||
if (repository.length === 0) throw new Error("github_release repository must have repository field");
|
||||
repoFix.owner = owner;
|
||||
repoFix.repository = repository.join("/");
|
||||
}
|
||||
if (repo.token) repoFix.token = repo.token;
|
||||
if (repo["apt-config"]) repoFix["apt-config"] = repo["apt-config"];
|
||||
if (repo.cronRefresh) repoFix.cronRefresh = repo.cronRefresh;
|
||||
return repoFix;
|
||||
}
|
||||
|
||||
return null;
|
||||
}).filter(a => !!a);
|
||||
return fixedConfig;
|
||||
}
|
||||
|
Reference in New Issue
Block a user