utils/node-apt
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 6 Wiki Activity

Docker image support #18

Merged
Sirherobrine23 merged 6 commits from dockerImage into main 2023-03-29 16:19:56 +00:00
Conversation 0 Commits 6 Files Changed 14 +364 -225
6 changed files with 364 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 095a20b58b - Show all commits

12
src/config.ts
View File

@ -211,7 +211,17 @@ export async function prettyConfig(tmpConfig: aptStreamConfig, optionsOverload?:
clientToken: data.clientToken,
gIds: (data.gIds ?? []).map(k => k?.trim()).filter(Boolean),
});
} else if (data.type === "docker") console.info("Ignore the docker image (%O), current not support Docker image, require more utils", data.image);
} else if (data.type === "docker") {
if (!data.image) throw new TypeError("misconfigured docker image, check your docker.image");
newConfigObject.repository[nName].source.push({
type: "docker",
componentName: data.componentName ?? null,
id,
image: data.image,
auth: data.auth ? {username: data.auth!.username, password: data.auth!.password} : undefined,
tags: data.tags instanceof Array ? data.tags.map(String) : [],
});
}
}
}

70
src/configManeger.ts
View File

@ -1,7 +1,8 @@
import * as dockerRegistry from "@sirherobrine23/docker-registry";
import { config, aptStreamConfig, save, repositorySource } from "./config.js";
import inquirer, { QuestionCollection } from "inquirer";
import { googleDriver, oracleBucket } from "@sirherobrine23/cloud";
import { loadRepository } from "./packageManege.js";
import { syncRepository } from "./packageManege.js";
import { connect } from "./database.js";
import { Github } from "@sirherobrine23/http";
import ora from "ora";
@ -44,8 +45,7 @@ async function createSource(): Promise<repositorySource> {
},
{
value: "docker",
name: "Docker or Open Container image (OCI) image",
disabled: true
name: "Docker or Open Container image (OCI) image"
}
]
});
@ -238,7 +238,64 @@ async function createSource(): Promise<repositorySource> {
})
};
} else if (target === "docker") {
console.log("Docker disabled, check in the future apt-stream support this");
const basicConfig = await inquirer.prompt<{authConfirm: boolean, imageURI: string}>([
{
name: "imageURI",
type: "input",
message: "Image URI/URL:",
validate(input) {
try {
new dockerRegistry.parseImage(input);
return true;
} catch (err) {
return String(err?.message || err);
}
},
},
{
name: "authConfirm",
type: "confirm",
message: "This registry or image required authentication?"
}
]);
let auth: dockerRegistry.userAuth;
if (basicConfig.authConfirm) {
const authPrompts = await inquirer.prompt([
{
name: "user",
type: "input",
message: "Username:",
validate(input: string) {
if (input.trim().length > 1) return true;
return "Invalid username";
}
},
{
name: "pass",
type: "password",
mask: "*",
message: "Password or Token:"
},
]);
auth = {
username: authPrompts.user,
password: authPrompts.pass
};
}
const registry = new dockerRegistry.v2(basicConfig.imageURI, auth);
const tags = await simpleQuestion<string[]>({
type: "checkbox",
message: "Select tags or don't select any to go to the last 6 tags at sync time",
choices: (await registry.getTags())
});
return {
type: "docker",
image: basicConfig.imageURI,
auth,
tags
};
}
console.log("Try again");
@ -319,7 +376,10 @@ export default async function main(configPath: string, configOld?: aptStreamConf
console.log("Saving...");
await save(configPath, localConfig);
console.log("Now loading all packages");
return loadRepository(await connect(localConfig), localConfig);
const sync = new syncRepository();
sync.on("error", console.error);
sync.on("addPackage", data => console.log("Added: %s -> %s/%s %s/%s", data.distName, data.componentName, data.control.Package, data.control.Version, data.control.Architecture, data.componentName));
return sync.sync(await connect(localConfig), localConfig);
}
return main(configPath, configOld);
}

26
src/database.ts
View File

@ -13,7 +13,7 @@ export interface packageData {
export interface packageManegerConfig {
getPackages(this: packageManeger): Promise<packageData[]>;
registryPackage?(this: packageManeger, distName: string, componentName: string, repoID: string, fileRestore: any, control: Debian.debianControl): Promise<{distName: string, componentName: string, packageName: string}>;
registryPackage?(this: packageManeger, ...args: Parameters<typeof packageManeger["prototype"]["addPackage"]>): ReturnType<typeof packageManeger["prototype"]["addPackage"]>;
findPackages?(this: packageManeger, search: {packageName?: string, packageArch?: string, packageComponent?: string, packageDist?: string}): Promise<packageData[]>;
}
@ -25,12 +25,18 @@ export class packageManeger {
}
search = async (search: {packageName?: string, packageArch?: string, packageComponent?: string, packageDist?: string}): ReturnType<typeof this.options.findPackages> => {
if (typeof this.options.findPackages !== "function") return (await this.getPackages()).filter(data => ((!search.packageName) || (search.packageName !== data.packageControl.Package)) && ((!search.packageArch) || (data.packageControl.Architecture !== search.packageArch)) && ((!search.packageComponent) || (data.packageComponent !== search.packageComponent)) && ((!search.packageDist) || (data.packageDistribuition !== search.packageDist)));
if (typeof this.options.findPackages !== "function") return (await this.getPackages()).filter(data => ((!search.packageName) || (search.packageName === data.packageControl.Package)) && ((!search.packageArch) || (data.packageControl.Architecture === search.packageArch)) && ((!search.packageComponent) || (data.packageComponent === search.packageComponent)) && ((!search.packageDist) || (data.packageDistribuition === search.packageDist)));
return this.options.findPackages.call(this, search);
}
addPackage = async (distName: string, componentName: string, repoID, fileRestore, control: Debian.debianControl): ReturnType<typeof this.options.registryPackage> => {
addPackage = async (distName: string, componentName: string, repoID: string, fileRestore: any, control: Debian.debianControl): Promise<{distName: string, componentName: string, control: Debian.debianControl}> => {
if (typeof this.options.registryPackage !== "function") throw new Error("Add package disabled");
if ((await this.search({
packageName: control.Package,
packageComponent: componentName,
packageArch: control.Architecture,
packageDist: distName
})).find(d => (d.packageControl.Version === control.Version))) throw new Error("Package exists!");
return this.options.registryPackage.call(this, distName, componentName, repoID, fileRestore, control);
}
}
@ -49,12 +55,6 @@ export async function connect(config: aptStreamConfig) {
},
async registryPackage(distName, componentName, repoID, fileRestore, control) {
if (!control) throw new Error("Error mal formado!");
if ((await this.search({
packageName: control.Package,
packageComponent: componentName,
packageArch: control.Architecture,
packageDist: distName
})).find(d => (d.packageControl.Version === control.Version))) throw new Error("Package exists!");
await collection.insertOne({
packageComponent: componentName,
packageDistribuition: distName,
@ -66,7 +66,7 @@ export async function connect(config: aptStreamConfig) {
return {
componentName,
distName,
packageName: control.Package
control,
};
}
});
@ -80,7 +80,6 @@ export async function connect(config: aptStreamConfig) {
return (await db.list({include_docs: true})).rows.map(data => data.doc);
},
async registryPackage(distName, componentName, repoID, fileRestore, control) {
if ((await this.search({packageName: control.Package, packageComponent: componentName, packageArch: control.Architecture})).find(d => (d.packageDistribuition === distName) && (d.packageControl.Version === control.Version))) throw new Error("Package exists!");
await db.insert({
packageDistribuition: distName,
packageComponent: componentName,
@ -92,7 +91,7 @@ export async function connect(config: aptStreamConfig) {
return {
componentName,
distName,
packageName: control.Package
control
};
},
});
@ -104,7 +103,6 @@ export async function connect(config: aptStreamConfig) {
return Array.from(packagesStorage);
},
async registryPackage(distName, componentName, repoID, fileRestore, control) {
if ((await this.search({packageName: control.Package, packageComponent: componentName, packageArch: control.Architecture})).find(d => (d.packageDistribuition === distName) && (d.packageControl.Version === control.Version))) throw new Error("Package exists!");
packagesStorage.push({
packageDistribuition: distName,
packageComponent: componentName,
@ -115,7 +113,7 @@ export async function connect(config: aptStreamConfig) {
return {
componentName,
distName,
packageName: control.Package
control
};
},
});

177
src/packageManege.ts
View File

@ -6,6 +6,7 @@ import { aptStreamConfig } from "./config.js";
import coreHttp, { Github } from "@sirherobrine23/http";
import stream from "stream";
import path from "node:path/posix";
import EventEmitter from "events";
export async function fileRestore(packageDb: packageData, repoConfig: aptStreamConfig): Promise<stream.Readable> {
const repo = repoConfig.repository[packageDb.packageDistribuition];
@ -35,96 +36,106 @@ export async function fileRestore(packageDb: packageData, repoConfig: aptStreamC
throw new Error("Check package type");
}
export async function loadRepository(packageManeger: packageManeger, config: aptStreamConfig, repository = Object.keys(config.repository)) {
const massaReturn: (Awaited<ReturnType<typeof packageManeger.addPackage>>)[] = []
for (const repo of repository || Object.keys(config.repository)) {
const source = config.repository[repo]?.source;
if (!source) continue;
for (const target of source) {
const { id } = target;
try {
if (target.type === "http") {
const { control } = await Debian.parsePackage(await coreHttp.streamRequest(target.url, {headers: target.auth?.header, query: target.auth?.query}));
massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control));
} else if (target.type === "oracle_bucket") {
const { authConfig, path = [] } = target;
const bucket = await oracleBucket.oracleBucket(authConfig);
if (path.length === 0) path.push(...((await bucket.listFiles()).filter(k => k.name.endsWith(".deb")).map(({name}) => name)));
await Promise.all(path.map(async file => {
const { control } = await Debian.parsePackage(await bucket.getFileStream(file));
return packageManeger.addPackage(repo, target.componentName || "main", id, {}, control);
})).then(d => massaReturn.push(...d));
} else if (target.type === "google_driver") {
const { clientId, clientSecret, clientToken, gIds = [] } = target;
const gdrive = await googleDriver.GoogleDriver({clientID: clientId, clientSecret, token: clientToken});
if (gIds.length === 0) gIds.push(...((await gdrive.listFiles()).filter(rel => rel.name.endsWith(".deb")).map(({id}) => id)));
await Promise.all(gIds.map(async file => {
const { control } = await Debian.parsePackage(await gdrive.getFileStream(file));
return packageManeger.addPackage(repo, target.componentName || "main", id, {}, control);
})).then(d => massaReturn.push(...d));
} else if (target.type === "github") {
const { owner, repository, token } = target;
const gh = await Github.GithubManeger(owner, repository, token);
if (target.subType === "branch") {
const { branch = (await gh.branchList()).at(0)?.name ?? "main" } = target;
for (const { path: filePath } of (await gh.trees(branch)).tree.filter(file => file.type === "tree" ? false : file["path"])) {
const rawURL = new URL(path.join(owner, repository, branch, filePath), "https://raw.githubusercontent.com");
const { control } = await Debian.parsePackage(await coreHttp.streamRequest(rawURL, {headers: token ? {Authorization: `token ${token}`} : {}}));
massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, {url: rawURL.toString()}, control));
}
} else {
const { tag = [] } = target;
await Promise.all(tag.map(async tagName => {
const assets = (await gh.getRelease(tagName)).assets.filter(({name}) => name.endsWith(".deb"));
for (const asset of assets) {
const { control } = await Debian.parsePackage(await coreHttp.streamRequest(asset.browser_download_url, {headers: token ? {Authorization: `token ${token}`} : {}}));
massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, {
url: asset.browser_download_url
}, control));
}
}));
}
} else if (target.type === "docker") {
const { image, auth, tags = [] } = target;
const registry = new dockerRegistry(image, auth);
if (tags.length === 0) {
const { sha256, tag } = registry.image;
if (sha256) tags.push(sha256);
else if (tag) tags.push(tag);
else tags.push(...((await registry.getTags()).reverse().slice(0, 6)));
}
const userAuth = new dockerAuth(registry.image, "pull", auth);
await userAuth.setup();
for (const tag of tags) {
const manifestManeger = new dockerUtils.Manifest(await registry.getManifets(tag, userAuth), registry);
const addPckage = async () => {
for (const layer of manifestManeger.getLayers()) {
const blob = await registry.extractLayer(layer.digest, layer.mediaType, userAuth);
blob.on("File", async entry => {
if (!(entry.path.endsWith(".deb"))) return null;
const { control } = await Debian.parsePackage(entry as any);
massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, {
ref: layer.digest,
path: entry.path,
export class syncRepository extends EventEmitter {
constructor() {
super({captureRejections: true});
}
}, control));
});
await new Promise<void>((done, reject) => blob.on("close", done).on("error", reject));
on(event: "error", fn: (err: any) => void): this;
on(event: "addPackage", fn: (data: Awaited<ReturnType<typeof packageManeger["prototype"]["addPackage"]>>) => void): this;
on(event: string, fn: (...args: any[]) => void) {
super.on(event, fn);
return this;
}
async sync(packageManeger: packageManeger, config: aptStreamConfig, repository = Object.keys(config.repository)) {
for (const repo of repository || Object.keys(config.repository)) {
const source = config.repository[repo]?.source;
if (!source) continue;
for (const target of source) {
const { id } = target;
try {
if (target.type === "http") {
const { control } = await Debian.parsePackage(await coreHttp.streamRequest(target.url, {headers: target.auth?.header, query: target.auth?.query}));
this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control));
} else if (target.type === "oracle_bucket") {
const { authConfig, path = [] } = target;
const bucket = await oracleBucket.oracleBucket(authConfig);
if (path.length === 0) path.push(...((await bucket.listFiles()).filter(k => k.name.endsWith(".deb")).map(({name}) => name)));
await Promise.all(path.map(async file => {
const { control } = await Debian.parsePackage(await bucket.getFileStream(file));
this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control));
}));
} else if (target.type === "google_driver") {
const { clientId, clientSecret, clientToken, gIds = [] } = target;
const gdrive = await googleDriver.GoogleDriver({clientID: clientId, clientSecret, token: clientToken});
if (gIds.length === 0) gIds.push(...((await gdrive.listFiles()).filter(rel => rel.name.endsWith(".deb")).map(({id}) => id)));
await Promise.all(gIds.map(async file => {
const { control } = await Debian.parsePackage(await gdrive.getFileStream(file));
this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control));
}));
} else if (target.type === "github") {
const { owner, repository, token } = target;
const gh = await Github.GithubManeger(owner, repository, token);
if (target.subType === "branch") {
const { branch = (await gh.branchList()).at(0)?.name ?? "main" } = target;
for (const { path: filePath } of (await gh.trees(branch)).tree.filter(file => file.type === "tree" ? false : file["path"])) {
const rawURL = new URL(path.join(owner, repository, branch, filePath), "https://raw.githubusercontent.com");
const { control } = await Debian.parsePackage(await coreHttp.streamRequest(rawURL, {headers: token ? {Authorization: `token ${token}`} : {}}));
this.emit("addPackage", (await packageManeger.addPackage(repo, target.componentName || "main", id, {url: rawURL.toString()}, control)));
}
} else {
const { tag = [] } = target;
for (const tagName of tag) {
const assets = (await gh.getRelease(tagName)).assets.filter(({name}) => name.endsWith(".deb"));
for (const asset of assets) {
const { control } = await Debian.parsePackage(await coreHttp.streamRequest(asset.browser_download_url, {headers: token ? {Authorization: `token ${token}`} : {}}));
this.emit("addPackage", (await packageManeger.addPackage(repo, target.componentName || "main", id, {url: asset.browser_download_url}, control)));
}
}
}
if (manifestManeger.multiArch) {
for (const platform of manifestManeger.platforms) {
await manifestManeger.setPlatform(platform as any);
await addPckage();
} else if (target.type === "docker") {
const { image, auth, tags = [] } = target;
const registry = new dockerRegistry(image, auth);
if (tags.length === 0) {
const { sha256, tag } = registry.image;
if (sha256) tags.push(sha256);
else if (tag) tags.push(tag);
else tags.push(...((await registry.getTags()).reverse().slice(0, 6)));
}
const userAuth = new dockerAuth(registry.image, "pull", auth);
await userAuth.setup();
for (const tag of tags) {
const manifestManeger = new dockerUtils.Manifest(await registry.getManifets(tag, userAuth), registry);
const addPckage = async () => {
for (const layer of manifestManeger.getLayers()) {
const blob = await registry.extractLayer(layer.digest, layer.mediaType, userAuth);
blob.on("File", async entry => {
if (!(entry.path.endsWith(".deb"))) return null;
console.log(entry.path);
try {
const { control } = await Debian.parsePackage(entry as any);
this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {
ref: layer.digest,
path: entry.path,
}, control));
} catch (err) {this.emit("error", err);}
});
await new Promise<void>((done, reject) => blob.on("close", done).on("error", reject));
}
}
} else await addPckage();
if (manifestManeger.multiArch) {
for (const platform of manifestManeger.platforms) {
await manifestManeger.setPlatform(platform as any);
await addPckage();
}
} else await addPckage();
}
}
} catch (err) {
this.emit("error", err);
}
} catch (err) {
console.error(err);
}
}
}
return massaReturn;
}