From cea866443aa847fd5adcfaccfb3f6d9f133ad27c Mon Sep 17 00:00:00 2001 From: Matheus Sampaio Queiroga Date: Mon, 27 Mar 2023 05:58:27 -0300 Subject: [PATCH 1/6] init support doxker image Signed-off-by: Matheus Sampaio Queiroga --- package-lock.json | 194 +++++++++++++++++++++++++++++----------------- package.json | 9 ++- 2 files changed, 127 insertions(+), 76 deletions(-) diff --git a/package-lock.json b/package-lock.json index 195ae9d..ce0003e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,10 +9,11 @@ "version": "1.0.0", "license": "GPL-2.0", "dependencies": { - "@sirherobrine23/cloud": "^3.4.3", - "@sirherobrine23/debian": "^3.4.3", - "@sirherobrine23/extends": "^3.4.3", - "@sirherobrine23/http": "^3.4.3", + "@sirherobrine23/cloud": "^3.5.0", + "@sirherobrine23/debian": "^3.5.0", + "@sirherobrine23/docker-registry": "^3.5.0", + "@sirherobrine23/extends": "^3.5.0", + "@sirherobrine23/http": "^3.5.0", "acme-client": "^5.0.0", "express": "^4.18.2", "inquirer": "^9.1.4", @@ -308,9 +309,9 @@ } }, "node_modules/@octokit/plugin-retry": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.2.tgz", - "integrity": "sha512-hscf7p/6DIQ8xbfDrMl9IflxugED6sFQvAUbSi75R6h/6hcNQgrb2vpfPTmyYKkdAEeTkUsEpzpQFdTAhSITOw==", + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.3.tgz", + "integrity": "sha512-3YKBj7d0J/4mpEc4xzMociWsMNl5lZqrpAnYcW6mqiSGF3wFjU+c6GHih6GLClk31JNvKDr0x9jc5cfm7evkZg==", "dependencies": { "@octokit/types": "^9.0.0", "bottleneck": "^2.15.3" @@ -413,17 +414,17 @@ } }, "node_modules/@sirherobrine23/ar": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/@sirherobrine23/ar/-/ar-3.4.3.tgz", - "integrity": "sha512-dXkriztjGK56PW57g8HX/nHojtUMDrFBpGgpUD5WiyzGiOzJx2nSN/B6FZhnTH+cQnhwO9hU0dsGBHQmFJya2w==" + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@sirherobrine23/ar/-/ar-3.5.0.tgz", + "integrity": "sha512-sB+CNnrxxX8uR1315pJIhHnYUii4WWVdquQdKj5xBuk9m+MdAMHZPBj960ld9aKLeccsPmr1IIkTUEHUUKOHHQ==" }, "node_modules/@sirherobrine23/cloud": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/@sirherobrine23/cloud/-/cloud-3.4.3.tgz", - "integrity": "sha512-cSaGanRbtC4y0XKv2cA2hUXLKsU0LdLh7515D/z0WvSwzrXT4lfPefoim+aOVDZb1s4mmuyNPmux2aUapeDKag==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@sirherobrine23/cloud/-/cloud-3.5.0.tgz", + "integrity": "sha512-6gZbbOKnmNodTnY1rJYY6by+R4v19AFrYQNbkJDH7hGJHT7+RsTohkIIzESBTnpmQzjyo+y+zhuo6VLPEtB4Lg==", "dependencies": { - "@sirherobrine23/extends": "3.4.3", - "@sirherobrine23/http": "3.4.3", + "@sirherobrine23/extends": "3.5.0", + "@sirherobrine23/http": "3.5.0", "chokidar": "^3.5.3", "googleapis": "^110.0.0", "oci-common": "^2.50.4", @@ -431,28 +432,36 @@ } }, "node_modules/@sirherobrine23/debian": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/@sirherobrine23/debian/-/debian-3.4.3.tgz", - "integrity": "sha512-SnhRdki15RPc6Q+rWJIiOm74i95H/RBGUs+pUBe9hCDIllbE+BI8vxo62GzO0d9BMt/ntaM4p5DPd5aCXubJIw==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@sirherobrine23/debian/-/debian-3.5.0.tgz", + "integrity": "sha512-doLoqfVyL/QNSqizrsOkZaO18ulUcx4jmEM0m2vnilDalR9IzjSzFL1nS930mXR3ZrUZf4h4IkjEXwsk0GaSDQ==", "dependencies": { - "@sirherobrine23/ar": "3.4.3", - "@sirherobrine23/extends": "3.4.3", - "@sirherobrine23/http": "3.4.3", + "@sirherobrine23/ar": "3.5.0", + "@sirherobrine23/extends": "3.5.0", + "@sirherobrine23/http": "3.5.0", "lzma-native": "^8.0.6", "unbzip2-stream": "^1.4.3" } }, + "node_modules/@sirherobrine23/docker-registry": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@sirherobrine23/docker-registry/-/docker-registry-3.5.0.tgz", + "integrity": "sha512-oJoVLS7eIr5bNzobORxDwtqMzleNmSsY5SFSaOwHGggEfFfLmtAg5TEUHskRNUulV7SgMtcTHWf3b1pZDDD9Ng==", + "dependencies": { + "@sirherobrine23/http": "3.5.0" + } + }, "node_modules/@sirherobrine23/extends": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/@sirherobrine23/extends/-/extends-3.4.3.tgz", - "integrity": "sha512-xrI/xIs9AihEOpzbkkGZsFmLszCMZ+jT27JOdUc+LW66ZYBWAqSGDQN/SeAytP1IC4eA1WnJvEppDESFPz2UrA==" + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@sirherobrine23/extends/-/extends-3.5.0.tgz", + "integrity": "sha512-FtYzfbc/6sqCODG/Nx7nK9B+zZWnvEM5hOnSiF36h18KqsR4mNYd+Zf1Heu3vQxxBBO0BTLlbh1KITHwLpAJeg==" }, "node_modules/@sirherobrine23/http": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/@sirherobrine23/http/-/http-3.4.3.tgz", - "integrity": "sha512-FSqbXweVpFb4YYHqod9KcOWubBy0V/7Glk4J79M90Fi/wh39hSBIdBYmnbMIJc+7f7hAHVAHCmK1qeadn75rgA==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@sirherobrine23/http/-/http-3.5.0.tgz", + "integrity": "sha512-0jXrEt7GLmKEr2MUE2x9GJOdYrYXyvaKYwI6WQ701CmTWJaPgwIso9VRqQxHEjbS2GjFyDJmhy89OeWHXKe8QQ==", "dependencies": { - "@sirherobrine23/extends": "3.4.3", + "@sirherobrine23/extends": "3.5.0", "acme-client": "^5.0.0", "adm-zip": "^0.5.10", "got": "^12.5.3", @@ -506,9 +515,9 @@ "dev": true }, "node_modules/@types/aws-lambda": { - "version": "8.10.111", - "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.111.tgz", - "integrity": "sha512-8HR9UjIKmoemEzE2BviVtFkeenxfbizSu8raFjnT2VXxguZZ2JTlNww7INOH7IA0J/zRa3TjOftkYq6hVNkxDA==" + "version": "8.10.114", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.114.tgz", + "integrity": "sha512-M8WpEGfC9iQ6V2Ccq6nGIXoQgeVc6z0Ngk8yCOL5V/TYIxshvb0MWQYLFFTZDesL0zmsoBc4OBjG9DB/4rei6w==" }, "node_modules/@types/body-parser": { "version": "1.19.2", @@ -1083,9 +1092,9 @@ } }, "node_modules/cacheable-request": { - "version": "10.2.8", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.8.tgz", - "integrity": "sha512-IDVO5MJ4LItE6HKFQTqT2ocAQsisOoCTUDu1ddCmnhyiwFQjXNPp4081Xj23N4tO+AFEFNzGuNEf/c8Gwwt15A==", + "version": "10.2.9", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.9.tgz", + "integrity": "sha512-CaAMr53AS1Tb9evO1BIWFnZjSr8A4pbXofpsNVWPMDZZj3ZQKHwsQG9BrTqQ4x5ZYJXz1T2b8LLtTZODxSpzbg==", "dependencies": { "@types/http-cache-semantics": "^4.0.1", "get-stream": "^6.0.1", @@ -1360,27 +1369,17 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, - "node_modules/cssom": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", - "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" - }, "node_modules/cssstyle": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", - "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-3.0.0.tgz", + "integrity": "sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==", "dependencies": { - "cssom": "~0.3.6" + "rrweb-cssom": "^0.6.0" }, "engines": { - "node": ">=8" + "node": ">=14" } }, - "node_modules/cssstyle/node_modules/cssom": { - "version": "0.3.8", - "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", - "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" - }, "node_modules/dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -1393,16 +1392,39 @@ } }, "node_modules/data-urls": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", - "integrity": "sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz", + "integrity": "sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==", "dependencies": { "abab": "^2.0.6", "whatwg-mimetype": "^3.0.0", - "whatwg-url": "^11.0.0" + "whatwg-url": "^12.0.0" }, "engines": { - "node": ">=12" + "node": ">=14" + } + }, + "node_modules/data-urls/node_modules/tr46": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz", + "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", + "dependencies": { + "punycode": "^2.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/data-urls/node_modules/whatwg-url": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-12.0.1.tgz", + "integrity": "sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==", + "dependencies": { + "tr46": "^4.1.1", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=14" } }, "node_modules/debug": { @@ -2062,14 +2084,14 @@ } }, "node_modules/got": { - "version": "12.5.3", - "resolved": "https://registry.npmjs.org/got/-/got-12.5.3.tgz", - "integrity": "sha512-8wKnb9MGU8IPGRIo+/ukTy9XLJBwDiCpIf5TVzQ9Cpol50eMTpBq2GAuDsuDIz7hTYmZgMgC1e9ydr6kSDWs3w==", + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-12.6.0.tgz", + "integrity": "sha512-WTcaQ963xV97MN3x0/CbAriXFZcXCfgxVp91I+Ze6pawQOa7SgzwSx2zIJJsX+kTajMnVs0xcFD1TxZKFqhdnQ==", "dependencies": { "@sindresorhus/is": "^5.2.0", "@szmarczak/http-timer": "^5.0.1", "cacheable-lookup": "^7.0.0", - "cacheable-request": "^10.2.1", + "cacheable-request": "^10.2.8", "decompress-response": "^6.0.0", "form-data-encoder": "^2.1.2", "get-stream": "^6.0.1", @@ -2389,17 +2411,16 @@ "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==" }, "node_modules/jsdom": { - "version": "21.1.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-21.1.0.tgz", - "integrity": "sha512-m0lzlP7qOtthD918nenK3hdItSd2I+V3W9IrBcB36sqDwG+KnUs66IF5GY7laGWUnlM9vTsD0W1QwSEBYWWcJg==", + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-21.1.1.tgz", + "integrity": "sha512-Jjgdmw48RKcdAIQyUD1UdBh2ecH7VqwaXPN3ehoZN6MqgVbMn+lRm1aAT1AsdJRAJpwfa4IpwgzySn61h2qu3w==", "dependencies": { "abab": "^2.0.6", - "acorn": "^8.8.1", + "acorn": "^8.8.2", "acorn-globals": "^7.0.0", - "cssom": "^0.5.0", - "cssstyle": "^2.3.0", - "data-urls": "^3.0.2", - "decimal.js": "^10.4.2", + "cssstyle": "^3.0.0", + "data-urls": "^4.0.0", + "decimal.js": "^10.4.3", "domexception": "^4.0.0", "escodegen": "^2.0.0", "form-data": "^4.0.0", @@ -2408,7 +2429,8 @@ "https-proxy-agent": "^5.0.1", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.2", - "parse5": "^7.1.1", + "parse5": "^7.1.2", + "rrweb-cssom": "^0.6.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^4.1.2", @@ -2416,8 +2438,8 @@ "webidl-conversions": "^7.0.0", "whatwg-encoding": "^2.0.0", "whatwg-mimetype": "^3.0.0", - "whatwg-url": "^11.0.0", - "ws": "^8.11.0", + "whatwg-url": "^12.0.1", + "ws": "^8.13.0", "xml-name-validator": "^4.0.0" }, "engines": { @@ -2432,6 +2454,29 @@ } } }, + "node_modules/jsdom/node_modules/tr46": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz", + "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", + "dependencies": { + "punycode": "^2.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/jsdom/node_modules/whatwg-url": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-12.0.1.tgz", + "integrity": "sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==", + "dependencies": { + "tr46": "^4.1.1", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -3290,6 +3335,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/rrweb-cssom": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz", + "integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==" + }, "node_modules/run-async": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", @@ -3964,9 +4014,9 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/ws": { - "version": "8.12.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.1.tgz", - "integrity": "sha512-1qo+M9Ba+xNhPB+YTWUlK6M17brTut5EXbcBaMRN5pH5dFrXz7lzz1ChFSUq3bOUl8yEvSenhHmYUNJxFzdJew==", + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", "engines": { "node": ">=10.0.0" }, diff --git a/package.json b/package.json index 4c56aff..5f851c6 100644 --- a/package.json +++ b/package.json @@ -44,10 +44,11 @@ "typescript": "^4.9.5" }, "dependencies": { - "@sirherobrine23/cloud": "^3.4.3", - "@sirherobrine23/debian": "^3.4.3", - "@sirherobrine23/extends": "^3.4.3", - "@sirherobrine23/http": "^3.4.3", + "@sirherobrine23/cloud": "^3.5.0", + "@sirherobrine23/debian": "^3.5.0", + "@sirherobrine23/docker-registry": "^3.5.0", + "@sirherobrine23/extends": "^3.5.0", + "@sirherobrine23/http": "^3.5.0", "acme-client": "^5.0.0", "express": "^4.18.2", "inquirer": "^9.1.4", -- 2.47.2 From 2892b4270c5364132ecfc3299a221efb22b92a84 Mon Sep 17 00:00:00 2001 From: Matheus Sampaio Queiroga Date: Mon, 27 Mar 2023 15:04:03 -0300 Subject: [PATCH 2/6] Init support to docker --- package-lock.json | 245 +++++++++++++++++++++---------------------- package.json | 10 +- src/config.ts | 9 +- src/packageManege.ts | 91 +++++++++------- 4 files changed, 182 insertions(+), 173 deletions(-) diff --git a/package-lock.json b/package-lock.json index ce0003e..e0897a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,11 +9,11 @@ "version": "1.0.0", "license": "GPL-2.0", "dependencies": { - "@sirherobrine23/cloud": "^3.5.0", - "@sirherobrine23/debian": "^3.5.0", - "@sirherobrine23/docker-registry": "^3.5.0", - "@sirherobrine23/extends": "^3.5.0", - "@sirherobrine23/http": "^3.5.0", + "@sirherobrine23/cloud": "^3.5.2", + "@sirherobrine23/debian": "^3.5.2", + "@sirherobrine23/docker-registry": "^3.5.2", + "@sirherobrine23/extends": "^3.5.2", + "@sirherobrine23/http": "^3.5.2", "acme-client": "^5.0.0", "express": "^4.18.2", "inquirer": "^9.1.4", @@ -414,17 +414,17 @@ } }, "node_modules/@sirherobrine23/ar": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/@sirherobrine23/ar/-/ar-3.5.0.tgz", - "integrity": "sha512-sB+CNnrxxX8uR1315pJIhHnYUii4WWVdquQdKj5xBuk9m+MdAMHZPBj960ld9aKLeccsPmr1IIkTUEHUUKOHHQ==" + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@sirherobrine23/ar/-/ar-3.5.2.tgz", + "integrity": "sha512-rvCyKgVUKp6XyWRBAP8hTfFtX2JQJOuWgccX0ICDyApSCwDvLq2pj4JqHppfYhi+g3rwLPgzNs++Sr1g4sb0eg==" }, "node_modules/@sirherobrine23/cloud": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/@sirherobrine23/cloud/-/cloud-3.5.0.tgz", - "integrity": "sha512-6gZbbOKnmNodTnY1rJYY6by+R4v19AFrYQNbkJDH7hGJHT7+RsTohkIIzESBTnpmQzjyo+y+zhuo6VLPEtB4Lg==", + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@sirherobrine23/cloud/-/cloud-3.5.2.tgz", + "integrity": "sha512-ym5AwSjekpuBc0kMmAbPnAjl4vNoTYFcWqIfb6BEpiYjJHdnSFAu2QWSPMIzUL9NOCsU8oiTLQVwkbj2Psxjyg==", "dependencies": { - "@sirherobrine23/extends": "3.5.0", - "@sirherobrine23/http": "3.5.0", + "@sirherobrine23/extends": "3.5.2", + "@sirherobrine23/http": "3.5.2", "chokidar": "^3.5.3", "googleapis": "^110.0.0", "oci-common": "^2.50.4", @@ -432,36 +432,37 @@ } }, "node_modules/@sirherobrine23/debian": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/@sirherobrine23/debian/-/debian-3.5.0.tgz", - "integrity": "sha512-doLoqfVyL/QNSqizrsOkZaO18ulUcx4jmEM0m2vnilDalR9IzjSzFL1nS930mXR3ZrUZf4h4IkjEXwsk0GaSDQ==", + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@sirherobrine23/debian/-/debian-3.5.2.tgz", + "integrity": "sha512-l2QJLnnEteGjkJLQbsxPlFyUCkJD6KxQnrudB1WQCEzC4q0E3zU/Ip7OSuZ0l5X0R6O8nhxgkEJJXGM0ZAeoPw==", "dependencies": { - "@sirherobrine23/ar": "3.5.0", - "@sirherobrine23/extends": "3.5.0", - "@sirherobrine23/http": "3.5.0", + "@sirherobrine23/ar": "3.5.2", + "@sirherobrine23/extends": "3.5.2", + "@sirherobrine23/http": "3.5.2", "lzma-native": "^8.0.6", "unbzip2-stream": "^1.4.3" } }, "node_modules/@sirherobrine23/docker-registry": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/@sirherobrine23/docker-registry/-/docker-registry-3.5.0.tgz", - "integrity": "sha512-oJoVLS7eIr5bNzobORxDwtqMzleNmSsY5SFSaOwHGggEfFfLmtAg5TEUHskRNUulV7SgMtcTHWf3b1pZDDD9Ng==", + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@sirherobrine23/docker-registry/-/docker-registry-3.5.2.tgz", + "integrity": "sha512-HKXfGRH7HkWJvJA2HRpbE4/E4Dl2FTEA7ePvw1v0LW/rgpWLlqqaxu1zcdtJUqjBLyn7q4n5x9YEe9BjSgD6ZA==", "dependencies": { - "@sirherobrine23/http": "3.5.0" + "@sirherobrine23/http": "3.5.2", + "tar": "^6.1.13" } }, "node_modules/@sirherobrine23/extends": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/@sirherobrine23/extends/-/extends-3.5.0.tgz", - "integrity": "sha512-FtYzfbc/6sqCODG/Nx7nK9B+zZWnvEM5hOnSiF36h18KqsR4mNYd+Zf1Heu3vQxxBBO0BTLlbh1KITHwLpAJeg==" + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@sirherobrine23/extends/-/extends-3.5.2.tgz", + "integrity": "sha512-sW/Utw2q/jzrWyS8XRcLOMBSsmfY7ft+sJlVqjN4uONdO3eLU1+VoAAulZg6If1gREJlkJUD6/XNm70e8xLpHw==" }, "node_modules/@sirherobrine23/http": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/@sirherobrine23/http/-/http-3.5.0.tgz", - "integrity": "sha512-0jXrEt7GLmKEr2MUE2x9GJOdYrYXyvaKYwI6WQ701CmTWJaPgwIso9VRqQxHEjbS2GjFyDJmhy89OeWHXKe8QQ==", + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@sirherobrine23/http/-/http-3.5.2.tgz", + "integrity": "sha512-apL3hVX7sqthp2kFDI9MtEBTHusOvVy+e+WOdkKBIRnbBSaNs+VV8jugFG/WFT6LzIUyVfZdZvDu9mYyUTM14A==", "dependencies": { - "@sirherobrine23/extends": "3.5.0", + "@sirherobrine23/extends": "3.5.2", "acme-client": "^5.0.0", "adm-zip": "^0.5.10", "got": "^12.5.3", @@ -620,9 +621,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.14.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.2.tgz", - "integrity": "sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==" + "version": "18.15.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.10.tgz", + "integrity": "sha512-9avDaQJczATcXgfmMAW3MIWArOO7A+m90vuCFLr8AotWf8igO/mRoYukrk2cqZVtv38tHs33retzHEilM7FpeQ==" }, "node_modules/@types/opossum": { "version": "4.1.1", @@ -702,9 +703,9 @@ } }, "node_modules/@types/yargs": { - "version": "17.0.22", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.22.tgz", - "integrity": "sha512-pet5WJ9U8yPVRhkwuEIp5ktAeAqRZOq4UdAyWLWzxbtpyXnzbtLdKiXAjJzi/KLmPGS9wk86lUFWZFN6sISo4g==", + "version": "17.0.23", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.23.tgz", + "integrity": "sha512-yuogunc04OnzGQCrfHx+Kk883Q4X0aSwmYZhKjI21m+SVYzjIbrWl8dOOwSv5hf2Um2pdCOXWo9isteZTNXUZQ==", "dev": true, "dependencies": { "@types/yargs-parser": "*" @@ -807,9 +808,9 @@ } }, "node_modules/ansi-escapes": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-6.0.0.tgz", - "integrity": "sha512-IG23inYII3dWlU2EyiAiGj6Bwal5GzsgPMwjYGvc1HPE2dgbj4ZB5ToWBKSquKw74nB3TIuOwaI6/jSULzfgrw==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-6.1.0.tgz", + "integrity": "sha512-bQyg9bzRntwR/8b89DOEhGwctcwCrbWW/TuqTQnpqpy5Fz3aovcOTj5i8NJV6AHc8OGNdMaqdxAWww8pz2kiKg==", "dependencies": { "type-fest": "^3.0.0" }, @@ -1035,9 +1036,9 @@ } }, "node_modules/bson": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/bson/-/bson-5.0.1.tgz", - "integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-5.1.0.tgz", + "integrity": "sha512-FEecNHkhYRBe7X9KDkdG12xNuz5VHGeH6mCE0B5sBmYtiR/Ux/9vUH/v4NUoBCDr6NuEhvahjoLiiRogptVW0A==", "engines": { "node": ">=14.20.1" } @@ -1404,29 +1405,6 @@ "node": ">=14" } }, - "node_modules/data-urls/node_modules/tr46": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz", - "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", - "dependencies": { - "punycode": "^2.3.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/data-urls/node_modules/whatwg-url": { - "version": "12.0.1", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-12.0.1.tgz", - "integrity": "sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==", - "dependencies": { - "tr46": "^4.1.1", - "webidl-conversions": "^7.0.0" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -2267,28 +2245,28 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/inquirer": { - "version": "9.1.4", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-9.1.4.tgz", - "integrity": "sha512-9hiJxE5gkK/cM2d1mTEnuurGTAoHebbkX0BYl3h7iEg7FYfuNIom+nDfBCSWtvSnoSrWCeBxqqBZu26xdlJlXA==", + "version": "9.1.5", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-9.1.5.tgz", + "integrity": "sha512-3ygAIh8gcZavV9bj6MTdYddG2zPSYswP808fKS46NOwlF0zZljVpnLCHODDqItWJDbDpLb3aouAxGaJbkxoppA==", "dependencies": { "ansi-escapes": "^6.0.0", - "chalk": "^5.1.2", + "chalk": "^5.2.0", "cli-cursor": "^4.0.0", "cli-width": "^4.0.0", "external-editor": "^3.0.3", "figures": "^5.0.0", "lodash": "^4.17.21", - "mute-stream": "0.0.8", + "mute-stream": "1.0.0", "ora": "^6.1.2", "run-async": "^2.4.0", - "rxjs": "^7.5.7", + "rxjs": "^7.8.0", "string-width": "^5.1.2", "strip-ansi": "^7.0.1", "through": "^2.3.6", - "wrap-ansi": "^8.0.1" + "wrap-ansi": "^8.1.0" }, "engines": { - "node": ">=12.0.0" + "node": ">=14.18.0" } }, "node_modules/ip": { @@ -2454,29 +2432,6 @@ } } }, - "node_modules/jsdom/node_modules/tr46": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz", - "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", - "dependencies": { - "punycode": "^2.3.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/jsdom/node_modules/whatwg-url": { - "version": "12.0.1", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-12.0.1.tgz", - "integrity": "sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==", - "dependencies": { - "tr46": "^4.1.1", - "webidl-conversions": "^7.0.0" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -2544,9 +2499,9 @@ } }, "node_modules/jsrsasign": { - "version": "10.6.1", - "resolved": "https://registry.npmjs.org/jsrsasign/-/jsrsasign-10.6.1.tgz", - "integrity": "sha512-emiQ05haY9CRj1Ho/LiuCqr/+8RgJuWdiHYNglIg2Qjfz0n+pnUq9I2QHplXuOMO2EnAW1oCGC1++aU5VoWSlw==", + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/jsrsasign/-/jsrsasign-10.7.0.tgz", + "integrity": "sha512-D5V2gGpYGtwbAtQHoglTVrpYf7QJuNoPEhaLOsTFONS2jXUl3qyR1hnYrNpASAybqQeiDYA3zGthR0ubgPRoQA==", "funding": { "url": "https://github.com/kjur/jsrsasign#donations" } @@ -2746,9 +2701,9 @@ "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" }, "node_modules/minipass": { - "version": "4.2.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.4.tgz", - "integrity": "sha512-lwycX3cBMTvcejsHITUgYj6Gy6A7Nh4Q6h9NP4sTHY1ccJlC7yKzDmiShEHsJ16Jf1nKGDEaiHxiltsJEvk0nQ==", + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.5.tgz", + "integrity": "sha512-+yQl7SX3bIT83Lhb4BVorMAHVuqsskxRdlmO9kTpyukp8vsm2Sn/fUOV9xlnG8/a5JsypJzap21lz/y3FBMJ8Q==", "engines": { "node": ">=8" } @@ -2828,15 +2783,41 @@ "whatwg-url": "^11.0.0" } }, + "node_modules/mongodb-connection-string-url/node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "dependencies": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/mute-stream": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } }, "node_modules/nano": { "version": "10.1.2", @@ -3102,17 +3083,17 @@ } }, "node_modules/ora": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/ora/-/ora-6.1.2.tgz", - "integrity": "sha512-EJQ3NiP5Xo94wJXIzAyOtSb0QEIAUu7m8t6UZ9krbz0vAJqr92JpcK/lEXg91q6B9pEGqrykkd2EQplnifDSBw==", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ora/-/ora-6.3.0.tgz", + "integrity": "sha512-1/D8uRFY0ay2kgBpmAwmSA404w4OoPVhHMqRqtjvrcK/dnzcEZxMJ+V4DUbyICu8IIVRclHcOf5wlD1tMY4GUQ==", "dependencies": { - "bl": "^5.0.0", "chalk": "^5.0.0", "cli-cursor": "^4.0.0", "cli-spinners": "^2.6.1", "is-interactive": "^2.0.0", "is-unicode-supported": "^1.1.0", "log-symbols": "^5.1.0", + "stdin-discarder": "^0.1.0", "strip-ansi": "^7.0.1", "wcwidth": "^1.0.1" }, @@ -3265,9 +3246,9 @@ } }, "node_modules/readable-stream": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", - "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -3567,6 +3548,20 @@ "node": ">= 0.8" } }, + "node_modules/stdin-discarder": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.1.0.tgz", + "integrity": "sha512-xhV7w8S+bUwlPTb4bAOUQhv8/cSS5offJuX8GQGq32ONF0ZtDWKfkdomM3HMRA+LhX6um/FZ0COqlwsjD53LeQ==", + "dependencies": { + "bl": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -3676,14 +3671,14 @@ } }, "node_modules/tr46": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", - "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz", + "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", "dependencies": { - "punycode": "^2.1.1" + "punycode": "^2.3.0" }, "engines": { - "node": ">=12" + "node": ">=14" } }, "node_modules/ts-node": { @@ -3751,9 +3746,9 @@ } }, "node_modules/type-fest": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.6.1.tgz", - "integrity": "sha512-htXWckxlT6U4+ilVgweNliPqlsVSSucbxVexRYllyMVJDtf5rTjv6kF/s+qAd4QSL1BZcnJPEJavYBPQiWuZDA==", + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.7.1.tgz", + "integrity": "sha512-8LZNdvuztgxCF4eYpEmPYUPS0lbbByM2qHcp2oMxHZhWLIQB9QE36EeQ1PKwsUIDZXEP8HCBEmkBbT1//kLU4Q==", "engines": { "node": ">=14.16" }, @@ -3973,15 +3968,15 @@ } }, "node_modules/whatwg-url": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", - "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-12.0.1.tgz", + "integrity": "sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==", "dependencies": { - "tr46": "^3.0.0", + "tr46": "^4.1.1", "webidl-conversions": "^7.0.0" }, "engines": { - "node": ">=12" + "node": ">=14" } }, "node_modules/word-wrap": { diff --git a/package.json b/package.json index 5f851c6..31dd81f 100644 --- a/package.json +++ b/package.json @@ -44,11 +44,11 @@ "typescript": "^4.9.5" }, "dependencies": { - "@sirherobrine23/cloud": "^3.5.0", - "@sirherobrine23/debian": "^3.5.0", - "@sirherobrine23/docker-registry": "^3.5.0", - "@sirherobrine23/extends": "^3.5.0", - "@sirherobrine23/http": "^3.5.0", + "@sirherobrine23/cloud": "^3.5.2", + "@sirherobrine23/debian": "^3.5.2", + "@sirherobrine23/docker-registry": "^3.5.2", + "@sirherobrine23/extends": "^3.5.2", + "@sirherobrine23/http": "^3.5.2", "acme-client": "^5.0.0", "express": "^4.18.2", "inquirer": "^9.1.4", diff --git a/src/config.ts b/src/config.ts index 9965087..b2e7409 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,6 @@ import { googleDriver, oracleBucket } from "@sirherobrine23/cloud"; import { extendsFS } from "@sirherobrine23/extends"; +import { userAuth } from "@sirherobrine23/docker-registry"; import fs from "node:fs/promises"; import yaml from "yaml"; import path from "node:path"; @@ -56,14 +57,10 @@ export type repositorySource = { path?: string[], authConfig: oracleBucket.oracleOptions }|{ - /** - * get files from Docker/OCI images - * - * @deprecated cannot load images current version, check latest APIs to get support - */ type: "docker", image: string, - auth?: any, + auth?: userAuth, + tags?: string[] }) export type aptStreamConfig = { diff --git a/src/packageManege.ts b/src/packageManege.ts index c19b011..751fbec 100644 --- a/src/packageManege.ts +++ b/src/packageManege.ts @@ -1,10 +1,11 @@ import * as Debian from "@sirherobrine23/debian"; +import { v2 as dockerRegistry, Auth as dockerAuth, Utils as dockerUtils } from "@sirherobrine23/docker-registry"; import { packageData, packageManeger } from "./database.js"; import { googleDriver, oracleBucket } from "@sirherobrine23/cloud"; import { aptStreamConfig } from "./config.js"; -import { extendsCrypto } from "@sirherobrine23/extends"; import coreHttp, { Github } from "@sirherobrine23/http"; import stream from "stream"; +import path from "node:path/posix"; export async function fileRestore(packageDb: packageData, repoConfig: aptStreamConfig): Promise { const repo = repoConfig.repository[packageDb.packageDistribuition]; @@ -26,16 +27,14 @@ export async function fileRestore(packageDb: packageData, repoConfig: aptStreamC const { clientId, clientSecret, clientToken } = source, { fileRestore: { id } } = packageDb; const gdrive = await googleDriver.GoogleDriver({clientID: clientId, clientSecret, token: clientToken}); return gdrive.getFileStream(id); + } else if (source.type === "docker") { + const { image, auth } = source, { ref, path: debPath } = packageDb.fileRestore; + const registry = new dockerRegistry(image, auth); + return new Promise((done, reject) => registry.extractLayer(ref).then(tar => tar.on("error", reject).on("File", entry => entry.path === debPath ? done(entry as any) : null))); } - throw new Error("Check package type"); } -async function genericParse(stream: stream.Readable) { - const hashs = extendsCrypto.createHashAsync(stream); - return Debian.parsePackage(stream).then(({control}) => hashs.then(hash => ({hash, control}))); -} - export async function loadRepository(packageManeger: packageManeger, config: aptStreamConfig, repository = Object.keys(config.repository)) { const massaReturn: (Awaited>)[] = [] for (const repo of repository || Object.keys(config.repository)) { @@ -45,24 +44,14 @@ export async function loadRepository(packageManeger: packageManeger, config: apt const { id } = target; try { if (target.type === "http") { - const { control, hash: { byteLength, hash } } = await genericParse(await coreHttp.streamRequest(target.url, {headers: target.auth?.header, query: target.auth?.query})); - control.Size = byteLength; - control.SHA512 = hash.sha512; - control.SHA256 = hash.sha256; - control.SHA1 = hash.sha1; - control.MD5sum = hash.md5; + const { control } = await Debian.parsePackage(await coreHttp.streamRequest(target.url, {headers: target.auth?.header, query: target.auth?.query})); massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)); } else if (target.type === "oracle_bucket") { const { authConfig, path = [] } = target; const bucket = await oracleBucket.oracleBucket(authConfig); - if (path.length === 0) path.push(...((await bucket.listFiles()).filter(k => k.name.endsWith(".dev")).map(({name}) => name))); + if (path.length === 0) path.push(...((await bucket.listFiles()).filter(k => k.name.endsWith(".deb")).map(({name}) => name))); await Promise.all(path.map(async file => { - const { control, hash: { byteLength, hash } } = await genericParse(await bucket.getFileStream(file)); - control.Size = byteLength; - control.SHA512 = hash.sha512; - control.SHA256 = hash.sha256; - control.SHA1 = hash.sha1; - control.MD5sum = hash.md5; + const { control } = await Debian.parsePackage(await bucket.getFileStream(file)); return packageManeger.addPackage(repo, target.componentName || "main", id, {}, control); })).then(d => massaReturn.push(...d)); } else if (target.type === "google_driver") { @@ -70,38 +59,66 @@ export async function loadRepository(packageManeger: packageManeger, config: apt const gdrive = await googleDriver.GoogleDriver({clientID: clientId, clientSecret, token: clientToken}); if (gIds.length === 0) gIds.push(...((await gdrive.listFiles()).filter(rel => rel.name.endsWith(".deb")).map(({id}) => id))); await Promise.all(gIds.map(async file => { - const { control, hash: { byteLength, hash } } = await genericParse(await gdrive.getFileStream(file)); - control.Size = byteLength; - control.SHA512 = hash.sha512; - control.SHA256 = hash.sha256; - control.SHA1 = hash.sha1; - control.MD5sum = hash.md5; + const { control } = await Debian.parsePackage(await gdrive.getFileStream(file)); return packageManeger.addPackage(repo, target.componentName || "main", id, {}, control); })).then(d => massaReturn.push(...d)); } else if (target.type === "github") { const { owner, repository, token } = target; const gh = await Github.GithubManeger(owner, repository, token); if (target.subType === "branch") { - const { branch = (await gh.branchList()).at(0).name } = target; - console.warn("Cannot load packages from %s/%s tree %O", owner, repository, branch); - // (await gh.trees(branch)); + const { branch = (await gh.branchList()).at(0)?.name ?? "main" } = target; + for (const { path: filePath } of (await gh.trees(branch)).tree.filter(file => file.type === "tree" ? false : file["path"])) { + const rawURL = new URL(path.join(owner, repository, branch, filePath), "https://raw.githubusercontent.com"); + const { control } = await Debian.parsePackage(await coreHttp.streamRequest(rawURL, {headers: token ? {Authorization: `token ${token}`} : {}})); + massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, {url: rawURL.toString()}, control)); + } } else { const { tag = [] } = target; await Promise.all(tag.map(async tagName => { const assets = (await gh.getRelease(tagName)).assets.filter(({name}) => name.endsWith(".deb")); for (const asset of assets) { - const { control, hash: { byteLength, hash } } = await genericParse(await coreHttp.streamRequest(asset.browser_download_url, {headers: token ? {Authorization: `token ${token}`} : {}})); - control.Size = byteLength; - control.SHA512 = hash.sha512; - control.SHA256 = hash.sha256; - control.SHA1 = hash.sha1; - control.MD5sum = hash.md5; - massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)); + const { control } = await Debian.parsePackage(await coreHttp.streamRequest(asset.browser_download_url, {headers: token ? {Authorization: `token ${token}`} : {}})); + massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, { + url: asset.browser_download_url + }, control)); } })); } } else if (target.type === "docker") { - console.warn("Current docker image is disabled!"); + const { image, auth, tags = [] } = target; + const registry = new dockerRegistry(image, auth); + if (tags.length === 0) { + const { sha256, tag } = registry.image; + if (sha256) tags.push(sha256); + else if (tag) tags.push(tag); + else tags.push(...((await registry.getTags()).reverse().slice(0, 6))); + } + const userAuth = new dockerAuth(registry.image, "pull", auth); + await userAuth.setup(); + for (const tag of tags) { + const manifestManeger = new dockerUtils.Manifest(await registry.getManifets(tag, userAuth), registry); + const addPckage = async () => { + for (const layer of manifestManeger.getLayers()) { + const blob = await registry.extractLayer(layer.digest, layer.mediaType, userAuth); + blob.on("File", async entry => { + if (!(entry.path.endsWith(".deb"))) return null; + const { control } = await Debian.parsePackage(entry as any); + massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, { + ref: layer.digest, + path: entry.path, + + }, control)); + }); + await new Promise((done, reject) => blob.on("close", done).on("error", reject)); + } + } + if (manifestManeger.multiArch) { + for (const platform of manifestManeger.platforms) { + await manifestManeger.setPlatform(platform as any); + await addPckage(); + } + } else await addPckage(); + } } } catch (err) { console.error(err); -- 2.47.2 From 095a20b58b54d25f0fe1bfac7fac1a10e9685c9b Mon Sep 17 00:00:00 2001 From: Matheus Sampaio Queiroga Date: Mon, 27 Mar 2023 19:07:56 -0300 Subject: [PATCH 3/6] Add docker config save and update docker parse --- src/config.ts | 12 ++- src/configManeger.ts | 70 +++++++++++++++-- src/database.ts | 26 +++---- src/packageManege.ts | 177 +++++++++++++++++++++++-------------------- 4 files changed, 182 insertions(+), 103 deletions(-) diff --git a/src/config.ts b/src/config.ts index b2e7409..85f652f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -211,7 +211,17 @@ export async function prettyConfig(tmpConfig: aptStreamConfig, optionsOverload?: clientToken: data.clientToken, gIds: (data.gIds ?? []).map(k => k?.trim()).filter(Boolean), }); - } else if (data.type === "docker") console.info("Ignore the docker image (%O), current not support Docker image, require more utils", data.image); + } else if (data.type === "docker") { + if (!data.image) throw new TypeError("misconfigured docker image, check your docker.image"); + newConfigObject.repository[nName].source.push({ + type: "docker", + componentName: data.componentName ?? null, + id, + image: data.image, + auth: data.auth ? {username: data.auth!.username, password: data.auth!.password} : undefined, + tags: data.tags instanceof Array ? data.tags.map(String) : [], + }); + } } } diff --git a/src/configManeger.ts b/src/configManeger.ts index 27bc101..5003d9f 100644 --- a/src/configManeger.ts +++ b/src/configManeger.ts @@ -1,7 +1,8 @@ +import * as dockerRegistry from "@sirherobrine23/docker-registry"; import { config, aptStreamConfig, save, repositorySource } from "./config.js"; import inquirer, { QuestionCollection } from "inquirer"; import { googleDriver, oracleBucket } from "@sirherobrine23/cloud"; -import { loadRepository } from "./packageManege.js"; +import { syncRepository } from "./packageManege.js"; import { connect } from "./database.js"; import { Github } from "@sirherobrine23/http"; import ora from "ora"; @@ -44,8 +45,7 @@ async function createSource(): Promise { }, { value: "docker", - name: "Docker or Open Container image (OCI) image", - disabled: true + name: "Docker or Open Container image (OCI) image" } ] }); @@ -238,7 +238,64 @@ async function createSource(): Promise { }) }; } else if (target === "docker") { - console.log("Docker disabled, check in the future apt-stream support this"); + const basicConfig = await inquirer.prompt<{authConfirm: boolean, imageURI: string}>([ + { + name: "imageURI", + type: "input", + message: "Image URI/URL:", + validate(input) { + try { + new dockerRegistry.parseImage(input); + return true; + } catch (err) { + return String(err?.message || err); + } + }, + }, + { + name: "authConfirm", + type: "confirm", + message: "This registry or image required authentication?" + } + ]); + let auth: dockerRegistry.userAuth; + if (basicConfig.authConfirm) { + const authPrompts = await inquirer.prompt([ + { + name: "user", + type: "input", + message: "Username:", + validate(input: string) { + if (input.trim().length > 1) return true; + return "Invalid username"; + } + }, + { + name: "pass", + type: "password", + mask: "*", + message: "Password or Token:" + }, + ]); + auth = { + username: authPrompts.user, + password: authPrompts.pass + }; + } + + const registry = new dockerRegistry.v2(basicConfig.imageURI, auth); + const tags = await simpleQuestion({ + type: "checkbox", + message: "Select tags or don't select any to go to the last 6 tags at sync time", + choices: (await registry.getTags()) + }); + + return { + type: "docker", + image: basicConfig.imageURI, + auth, + tags + }; } console.log("Try again"); @@ -319,7 +376,10 @@ export default async function main(configPath: string, configOld?: aptStreamConf console.log("Saving..."); await save(configPath, localConfig); console.log("Now loading all packages"); - return loadRepository(await connect(localConfig), localConfig); + const sync = new syncRepository(); + sync.on("error", console.error); + sync.on("addPackage", data => console.log("Added: %s -> %s/%s %s/%s", data.distName, data.componentName, data.control.Package, data.control.Version, data.control.Architecture, data.componentName)); + return sync.sync(await connect(localConfig), localConfig); } return main(configPath, configOld); } diff --git a/src/database.ts b/src/database.ts index 3778e07..475c6a7 100644 --- a/src/database.ts +++ b/src/database.ts @@ -13,7 +13,7 @@ export interface packageData { export interface packageManegerConfig { getPackages(this: packageManeger): Promise; - registryPackage?(this: packageManeger, distName: string, componentName: string, repoID: string, fileRestore: any, control: Debian.debianControl): Promise<{distName: string, componentName: string, packageName: string}>; + registryPackage?(this: packageManeger, ...args: Parameters): ReturnType; findPackages?(this: packageManeger, search: {packageName?: string, packageArch?: string, packageComponent?: string, packageDist?: string}): Promise; } @@ -25,12 +25,18 @@ export class packageManeger { } search = async (search: {packageName?: string, packageArch?: string, packageComponent?: string, packageDist?: string}): ReturnType => { - if (typeof this.options.findPackages !== "function") return (await this.getPackages()).filter(data => ((!search.packageName) || (search.packageName !== data.packageControl.Package)) && ((!search.packageArch) || (data.packageControl.Architecture !== search.packageArch)) && ((!search.packageComponent) || (data.packageComponent !== search.packageComponent)) && ((!search.packageDist) || (data.packageDistribuition !== search.packageDist))); + if (typeof this.options.findPackages !== "function") return (await this.getPackages()).filter(data => ((!search.packageName) || (search.packageName === data.packageControl.Package)) && ((!search.packageArch) || (data.packageControl.Architecture === search.packageArch)) && ((!search.packageComponent) || (data.packageComponent === search.packageComponent)) && ((!search.packageDist) || (data.packageDistribuition === search.packageDist))); return this.options.findPackages.call(this, search); } - addPackage = async (distName: string, componentName: string, repoID, fileRestore, control: Debian.debianControl): ReturnType => { + addPackage = async (distName: string, componentName: string, repoID: string, fileRestore: any, control: Debian.debianControl): Promise<{distName: string, componentName: string, control: Debian.debianControl}> => { if (typeof this.options.registryPackage !== "function") throw new Error("Add package disabled"); + if ((await this.search({ + packageName: control.Package, + packageComponent: componentName, + packageArch: control.Architecture, + packageDist: distName + })).find(d => (d.packageControl.Version === control.Version))) throw new Error("Package exists!"); return this.options.registryPackage.call(this, distName, componentName, repoID, fileRestore, control); } } @@ -49,12 +55,6 @@ export async function connect(config: aptStreamConfig) { }, async registryPackage(distName, componentName, repoID, fileRestore, control) { if (!control) throw new Error("Error mal formado!"); - if ((await this.search({ - packageName: control.Package, - packageComponent: componentName, - packageArch: control.Architecture, - packageDist: distName - })).find(d => (d.packageControl.Version === control.Version))) throw new Error("Package exists!"); await collection.insertOne({ packageComponent: componentName, packageDistribuition: distName, @@ -66,7 +66,7 @@ export async function connect(config: aptStreamConfig) { return { componentName, distName, - packageName: control.Package + control, }; } }); @@ -80,7 +80,6 @@ export async function connect(config: aptStreamConfig) { return (await db.list({include_docs: true})).rows.map(data => data.doc); }, async registryPackage(distName, componentName, repoID, fileRestore, control) { - if ((await this.search({packageName: control.Package, packageComponent: componentName, packageArch: control.Architecture})).find(d => (d.packageDistribuition === distName) && (d.packageControl.Version === control.Version))) throw new Error("Package exists!"); await db.insert({ packageDistribuition: distName, packageComponent: componentName, @@ -92,7 +91,7 @@ export async function connect(config: aptStreamConfig) { return { componentName, distName, - packageName: control.Package + control }; }, }); @@ -104,7 +103,6 @@ export async function connect(config: aptStreamConfig) { return Array.from(packagesStorage); }, async registryPackage(distName, componentName, repoID, fileRestore, control) { - if ((await this.search({packageName: control.Package, packageComponent: componentName, packageArch: control.Architecture})).find(d => (d.packageDistribuition === distName) && (d.packageControl.Version === control.Version))) throw new Error("Package exists!"); packagesStorage.push({ packageDistribuition: distName, packageComponent: componentName, @@ -115,7 +113,7 @@ export async function connect(config: aptStreamConfig) { return { componentName, distName, - packageName: control.Package + control }; }, }); diff --git a/src/packageManege.ts b/src/packageManege.ts index 751fbec..b737495 100644 --- a/src/packageManege.ts +++ b/src/packageManege.ts @@ -6,6 +6,7 @@ import { aptStreamConfig } from "./config.js"; import coreHttp, { Github } from "@sirherobrine23/http"; import stream from "stream"; import path from "node:path/posix"; +import EventEmitter from "events"; export async function fileRestore(packageDb: packageData, repoConfig: aptStreamConfig): Promise { const repo = repoConfig.repository[packageDb.packageDistribuition]; @@ -35,96 +36,106 @@ export async function fileRestore(packageDb: packageData, repoConfig: aptStreamC throw new Error("Check package type"); } -export async function loadRepository(packageManeger: packageManeger, config: aptStreamConfig, repository = Object.keys(config.repository)) { - const massaReturn: (Awaited>)[] = [] - for (const repo of repository || Object.keys(config.repository)) { - const source = config.repository[repo]?.source; - if (!source) continue; - for (const target of source) { - const { id } = target; - try { - if (target.type === "http") { - const { control } = await Debian.parsePackage(await coreHttp.streamRequest(target.url, {headers: target.auth?.header, query: target.auth?.query})); - massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)); - } else if (target.type === "oracle_bucket") { - const { authConfig, path = [] } = target; - const bucket = await oracleBucket.oracleBucket(authConfig); - if (path.length === 0) path.push(...((await bucket.listFiles()).filter(k => k.name.endsWith(".deb")).map(({name}) => name))); - await Promise.all(path.map(async file => { - const { control } = await Debian.parsePackage(await bucket.getFileStream(file)); - return packageManeger.addPackage(repo, target.componentName || "main", id, {}, control); - })).then(d => massaReturn.push(...d)); - } else if (target.type === "google_driver") { - const { clientId, clientSecret, clientToken, gIds = [] } = target; - const gdrive = await googleDriver.GoogleDriver({clientID: clientId, clientSecret, token: clientToken}); - if (gIds.length === 0) gIds.push(...((await gdrive.listFiles()).filter(rel => rel.name.endsWith(".deb")).map(({id}) => id))); - await Promise.all(gIds.map(async file => { - const { control } = await Debian.parsePackage(await gdrive.getFileStream(file)); - return packageManeger.addPackage(repo, target.componentName || "main", id, {}, control); - })).then(d => massaReturn.push(...d)); - } else if (target.type === "github") { - const { owner, repository, token } = target; - const gh = await Github.GithubManeger(owner, repository, token); - if (target.subType === "branch") { - const { branch = (await gh.branchList()).at(0)?.name ?? "main" } = target; - for (const { path: filePath } of (await gh.trees(branch)).tree.filter(file => file.type === "tree" ? false : file["path"])) { - const rawURL = new URL(path.join(owner, repository, branch, filePath), "https://raw.githubusercontent.com"); - const { control } = await Debian.parsePackage(await coreHttp.streamRequest(rawURL, {headers: token ? {Authorization: `token ${token}`} : {}})); - massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, {url: rawURL.toString()}, control)); - } - } else { - const { tag = [] } = target; - await Promise.all(tag.map(async tagName => { - const assets = (await gh.getRelease(tagName)).assets.filter(({name}) => name.endsWith(".deb")); - for (const asset of assets) { - const { control } = await Debian.parsePackage(await coreHttp.streamRequest(asset.browser_download_url, {headers: token ? {Authorization: `token ${token}`} : {}})); - massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, { - url: asset.browser_download_url - }, control)); - } - })); - } - } else if (target.type === "docker") { - const { image, auth, tags = [] } = target; - const registry = new dockerRegistry(image, auth); - if (tags.length === 0) { - const { sha256, tag } = registry.image; - if (sha256) tags.push(sha256); - else if (tag) tags.push(tag); - else tags.push(...((await registry.getTags()).reverse().slice(0, 6))); - } - const userAuth = new dockerAuth(registry.image, "pull", auth); - await userAuth.setup(); - for (const tag of tags) { - const manifestManeger = new dockerUtils.Manifest(await registry.getManifets(tag, userAuth), registry); - const addPckage = async () => { - for (const layer of manifestManeger.getLayers()) { - const blob = await registry.extractLayer(layer.digest, layer.mediaType, userAuth); - blob.on("File", async entry => { - if (!(entry.path.endsWith(".deb"))) return null; - const { control } = await Debian.parsePackage(entry as any); - massaReturn.push(await packageManeger.addPackage(repo, target.componentName || "main", id, { - ref: layer.digest, - path: entry.path, +export class syncRepository extends EventEmitter { + constructor() { + super({captureRejections: true}); + } - }, control)); - }); - await new Promise((done, reject) => blob.on("close", done).on("error", reject)); + on(event: "error", fn: (err: any) => void): this; + on(event: "addPackage", fn: (data: Awaited>) => void): this; + on(event: string, fn: (...args: any[]) => void) { + super.on(event, fn); + return this; + } + + async sync(packageManeger: packageManeger, config: aptStreamConfig, repository = Object.keys(config.repository)) { + for (const repo of repository || Object.keys(config.repository)) { + const source = config.repository[repo]?.source; + if (!source) continue; + for (const target of source) { + const { id } = target; + try { + if (target.type === "http") { + const { control } = await Debian.parsePackage(await coreHttp.streamRequest(target.url, {headers: target.auth?.header, query: target.auth?.query})); + this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)); + } else if (target.type === "oracle_bucket") { + const { authConfig, path = [] } = target; + const bucket = await oracleBucket.oracleBucket(authConfig); + if (path.length === 0) path.push(...((await bucket.listFiles()).filter(k => k.name.endsWith(".deb")).map(({name}) => name))); + await Promise.all(path.map(async file => { + const { control } = await Debian.parsePackage(await bucket.getFileStream(file)); + this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)); + })); + } else if (target.type === "google_driver") { + const { clientId, clientSecret, clientToken, gIds = [] } = target; + const gdrive = await googleDriver.GoogleDriver({clientID: clientId, clientSecret, token: clientToken}); + if (gIds.length === 0) gIds.push(...((await gdrive.listFiles()).filter(rel => rel.name.endsWith(".deb")).map(({id}) => id))); + await Promise.all(gIds.map(async file => { + const { control } = await Debian.parsePackage(await gdrive.getFileStream(file)); + this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)); + })); + } else if (target.type === "github") { + const { owner, repository, token } = target; + const gh = await Github.GithubManeger(owner, repository, token); + if (target.subType === "branch") { + const { branch = (await gh.branchList()).at(0)?.name ?? "main" } = target; + for (const { path: filePath } of (await gh.trees(branch)).tree.filter(file => file.type === "tree" ? false : file["path"])) { + const rawURL = new URL(path.join(owner, repository, branch, filePath), "https://raw.githubusercontent.com"); + const { control } = await Debian.parsePackage(await coreHttp.streamRequest(rawURL, {headers: token ? {Authorization: `token ${token}`} : {}})); + this.emit("addPackage", (await packageManeger.addPackage(repo, target.componentName || "main", id, {url: rawURL.toString()}, control))); + } + } else { + const { tag = [] } = target; + for (const tagName of tag) { + const assets = (await gh.getRelease(tagName)).assets.filter(({name}) => name.endsWith(".deb")); + for (const asset of assets) { + const { control } = await Debian.parsePackage(await coreHttp.streamRequest(asset.browser_download_url, {headers: token ? {Authorization: `token ${token}`} : {}})); + this.emit("addPackage", (await packageManeger.addPackage(repo, target.componentName || "main", id, {url: asset.browser_download_url}, control))); + } } } - if (manifestManeger.multiArch) { - for (const platform of manifestManeger.platforms) { - await manifestManeger.setPlatform(platform as any); - await addPckage(); + } else if (target.type === "docker") { + const { image, auth, tags = [] } = target; + const registry = new dockerRegistry(image, auth); + if (tags.length === 0) { + const { sha256, tag } = registry.image; + if (sha256) tags.push(sha256); + else if (tag) tags.push(tag); + else tags.push(...((await registry.getTags()).reverse().slice(0, 6))); + } + const userAuth = new dockerAuth(registry.image, "pull", auth); + await userAuth.setup(); + for (const tag of tags) { + const manifestManeger = new dockerUtils.Manifest(await registry.getManifets(tag, userAuth), registry); + const addPckage = async () => { + for (const layer of manifestManeger.getLayers()) { + const blob = await registry.extractLayer(layer.digest, layer.mediaType, userAuth); + blob.on("File", async entry => { + if (!(entry.path.endsWith(".deb"))) return null; + console.log(entry.path); + try { + const { control } = await Debian.parsePackage(entry as any); + this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, { + ref: layer.digest, + path: entry.path, + }, control)); + } catch (err) {this.emit("error", err);} + }); + await new Promise((done, reject) => blob.on("close", done).on("error", reject)); + } } - } else await addPckage(); + if (manifestManeger.multiArch) { + for (const platform of manifestManeger.platforms) { + await manifestManeger.setPlatform(platform as any); + await addPckage(); + } + } else await addPckage(); + } } + } catch (err) { + this.emit("error", err); } - } catch (err) { - console.error(err); } } } - - return massaReturn; } \ No newline at end of file -- 2.47.2 From be1e8e7ebf03f9dd25cce369a638efa05086db4c Mon Sep 17 00:00:00 2001 From: Matheus Sampaio Queiroga Date: Tue, 28 Mar 2023 14:06:20 -0300 Subject: [PATCH 4/6] Catch all erros and continue listening files if not exsists in database --- package-lock.json | 223 ++++++++++++++++++++++++++++++++++--------- package.json | 20 ++-- src/packageManege.ts | 70 ++++++++------ 3 files changed, 225 insertions(+), 88 deletions(-) diff --git a/package-lock.json b/package-lock.json index e0897a9..a74250a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,19 +9,19 @@ "version": "1.0.0", "license": "GPL-2.0", "dependencies": { - "@sirherobrine23/cloud": "^3.5.2", - "@sirherobrine23/debian": "^3.5.2", - "@sirherobrine23/docker-registry": "^3.5.2", - "@sirherobrine23/extends": "^3.5.2", - "@sirherobrine23/http": "^3.5.2", + "@sirherobrine23/cloud": "^3.5.4", + "@sirherobrine23/debian": "^3.5.4", + "@sirherobrine23/docker-registry": "^3.5.4", + "@sirherobrine23/extends": "^3.5.4", + "@sirherobrine23/http": "^3.5.4", "acme-client": "^5.0.0", "express": "^4.18.2", - "inquirer": "^9.1.4", + "inquirer": "^9.1.5", "lzma-native": "^8.0.6", "mongodb": "^5.1.0", "nano": "^10.1.2", "openpgp": "^5.7.0", - "ora": "^6.1.2", + "ora": "^6.3.0", "semver": "^7.3.8", "tar": "^6.1.13", "yaml": "^2.2.1", @@ -34,12 +34,12 @@ "@types/express": "^4.17.17", "@types/inquirer": "^9.0.3", "@types/lzma-native": "^4.0.1", - "@types/node": "^18.14.2", + "@types/node": "^18.15.10", "@types/semver": "^7.3.13", "@types/tar": "^6.1.4", - "@types/yargs": "^17.0.22", + "@types/yargs": "^17.0.24", "ts-node": "^10.9.1", - "typescript": "^4.9.5" + "typescript": "^5.0.2" }, "engines": { "node": ">=14.0.0" @@ -414,59 +414,71 @@ } }, "node_modules/@sirherobrine23/ar": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@sirherobrine23/ar/-/ar-3.5.2.tgz", - "integrity": "sha512-rvCyKgVUKp6XyWRBAP8hTfFtX2JQJOuWgccX0ICDyApSCwDvLq2pj4JqHppfYhi+g3rwLPgzNs++Sr1g4sb0eg==" + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@sirherobrine23/ar/-/ar-3.5.4.tgz", + "integrity": "sha512-9/01fOwIvUh35MnsPxSE6gCBq5VHZzmpEyqJKG7gjUvWKb4sqw6onAlbgKFN5tGw5xt5lXBHMWV+ZF7RiFr04A==" }, "node_modules/@sirherobrine23/cloud": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@sirherobrine23/cloud/-/cloud-3.5.2.tgz", - "integrity": "sha512-ym5AwSjekpuBc0kMmAbPnAjl4vNoTYFcWqIfb6BEpiYjJHdnSFAu2QWSPMIzUL9NOCsU8oiTLQVwkbj2Psxjyg==", + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@sirherobrine23/cloud/-/cloud-3.5.4.tgz", + "integrity": "sha512-zODsjbKMOS9vGJqfLCK101GALPtejDtotjwZuy43QOO9tys3s4ASmW18vNaxXhoKB+OHiBYffpa/88OupOGT2Q==", "dependencies": { - "@sirherobrine23/extends": "3.5.2", - "@sirherobrine23/http": "3.5.2", + "@sirherobrine23/extends": "3.5.4", + "@sirherobrine23/http": "3.5.4", "chokidar": "^3.5.3", "googleapis": "^110.0.0", "oci-common": "^2.50.4", - "oci-objectstorage": "^2.53.1" + "oci-objectstorage": "^2.55.0" } }, "node_modules/@sirherobrine23/debian": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@sirherobrine23/debian/-/debian-3.5.2.tgz", - "integrity": "sha512-l2QJLnnEteGjkJLQbsxPlFyUCkJD6KxQnrudB1WQCEzC4q0E3zU/Ip7OSuZ0l5X0R6O8nhxgkEJJXGM0ZAeoPw==", + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@sirherobrine23/debian/-/debian-3.5.4.tgz", + "integrity": "sha512-Uk+XiyhoAKox2PpTZ6Db7sHC3LMRoetnk/GVONuup6LsIXt8U6nbgpwb+jjxTaoujleUj8opZ+zimk3LqTyETg==", "dependencies": { - "@sirherobrine23/ar": "3.5.2", - "@sirherobrine23/extends": "3.5.2", - "@sirherobrine23/http": "3.5.2", + "@sirherobrine23/ar": "3.5.4", + "@sirherobrine23/extends": "3.5.4", + "@sirherobrine23/http": "3.5.4", "lzma-native": "^8.0.6", "unbzip2-stream": "^1.4.3" } }, - "node_modules/@sirherobrine23/docker-registry": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@sirherobrine23/docker-registry/-/docker-registry-3.5.2.tgz", - "integrity": "sha512-HKXfGRH7HkWJvJA2HRpbE4/E4Dl2FTEA7ePvw1v0LW/rgpWLlqqaxu1zcdtJUqjBLyn7q4n5x9YEe9BjSgD6ZA==", + "node_modules/@sirherobrine23/decompress": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@sirherobrine23/decompress/-/decompress-3.5.4.tgz", + "integrity": "sha512-eavUtMdjMN5oYt0fkuTQ5UShags6lPNR/UHXUPkO0/pLI+kCweffVrYNpGtF086r4Er34c+peLvQqobBaFwxZQ==", "dependencies": { - "@sirherobrine23/http": "3.5.2", - "tar": "^6.1.13" + "duplexify": "^4.1.2" + }, + "optionalDependencies": { + "lzma-native": "^8.0.6" + } + }, + "node_modules/@sirherobrine23/docker-registry": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@sirherobrine23/docker-registry/-/docker-registry-3.5.4.tgz", + "integrity": "sha512-78Ihoho1on4MWom7juiqvX7cQnDPWyfq/edKpW24++JI/fNS9+R1YDjT6RqyAjAvSMDpLUrjKPjlsglZvKvzmA==", + "dependencies": { + "@sirherobrine23/decompress": "3.5.4", + "@sirherobrine23/http": "3.5.4", + "tar-stream": "^3.0.0" } }, "node_modules/@sirherobrine23/extends": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@sirherobrine23/extends/-/extends-3.5.2.tgz", - "integrity": "sha512-sW/Utw2q/jzrWyS8XRcLOMBSsmfY7ft+sJlVqjN4uONdO3eLU1+VoAAulZg6If1gREJlkJUD6/XNm70e8xLpHw==" + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@sirherobrine23/extends/-/extends-3.5.4.tgz", + "integrity": "sha512-qiRph46W7hK/eeoWwgUTJzOjW7Wp8vM1OVj/mjL+ZOVV66Q0RL06vWghIYVyoy5+gbWQJ0OuE+keO+xjkhfkRw==" }, "node_modules/@sirherobrine23/http": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@sirherobrine23/http/-/http-3.5.2.tgz", - "integrity": "sha512-apL3hVX7sqthp2kFDI9MtEBTHusOvVy+e+WOdkKBIRnbBSaNs+VV8jugFG/WFT6LzIUyVfZdZvDu9mYyUTM14A==", + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/@sirherobrine23/http/-/http-3.5.4.tgz", + "integrity": "sha512-EjEZ8dKUKVd+GtDXUM08RE6LUoFA1JLUKbbNNqs1e0A11tsNghmtwCbEdJRYiHCc9HtAKNRmt8gWtT/ngWr5cg==", "dependencies": { - "@sirherobrine23/extends": "3.5.2", + "@sirherobrine23/extends": "3.5.4", "acme-client": "^5.0.0", "adm-zip": "^0.5.10", - "got": "^12.5.3", - "jsdom": "^21.1.0", + "got": "^12.6.0", + "jsdom": "^21.1.1", "octokit": "^2.0.14", "tar": "^6.1.13", "yaml": "^2.2.1" @@ -703,9 +715,9 @@ } }, "node_modules/@types/yargs": { - "version": "17.0.23", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.23.tgz", - "integrity": "sha512-yuogunc04OnzGQCrfHx+Kk883Q4X0aSwmYZhKjI21m+SVYzjIbrWl8dOOwSv5hf2Um2pdCOXWo9isteZTNXUZQ==", + "version": "17.0.24", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", + "integrity": "sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==", "dev": true, "dependencies": { "@types/yargs-parser": "*" @@ -722,6 +734,17 @@ "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==" }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -920,6 +943,11 @@ "form-data": "^4.0.0" } }, + "node_modules/b4a": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.3.tgz", + "integrity": "sha512-aX6/FqpWQve8VN9kyTExy7GlmwNShvxcCWWD5QVR3ZbRlyBGtCrG5Autu95xxSPH4CRs+5PSV4d7PRnWpmqFlA==" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1525,6 +1553,17 @@ "node": ">=12" } }, + "node_modules/duplexify": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz", + "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -1565,6 +1604,14 @@ "node": ">= 0.8" } }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/entities": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz", @@ -1662,6 +1709,22 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/express": { "version": "4.18.2", "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", @@ -1742,6 +1805,11 @@ "node >=0.6.0" ] }, + "node_modules/fast-fifo": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.1.0.tgz", + "integrity": "sha512-Kl29QoNbNvn4nhDsLYjyIAaIqaJB6rBx5p3sL9VjaefJ+eMFBWVZiaoguaoZfzEKr5RhAti0UgM8703akGPJ6g==" + }, "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", @@ -3163,6 +3231,14 @@ "node": ">= 0.8.0" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -3212,6 +3288,11 @@ "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -3562,6 +3643,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/stream-shift": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" + }, + "node_modules/streamx": { + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.13.2.tgz", + "integrity": "sha512-+TWqixPhGDXEG9L/XczSbhfkmwAtGs3BJX5QNU6cvno+pOLKeszByWcnaTu6dg8efsTYqR8ZZuXWHhZfgrxMvA==", + "dependencies": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -3621,6 +3716,40 @@ "node": ">=10" } }, + "node_modules/tar-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.0.0.tgz", + "integrity": "sha512-O6OfUKBbQOqAhh6owTWmA730J/yZCYcpmZ1DBj2YX51ZQrt7d7NgzrR+CnO9wP6nt/viWZW2XeXLavX3/ZEbEg==", + "dependencies": { + "b4a": "^1.6.1", + "bl": "^6.0.0", + "streamx": "^2.12.5" + } + }, + "node_modules/tar-stream/node_modules/bl": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/bl/-/bl-6.0.1.tgz", + "integrity": "sha512-zk1P1eAEBHhhB+4NfGxqmuV6NgwECnIoRgsOq2ObdEsmoFVIYzJ/Jjcgaj7JOY/8ekH27bIHSV4Si2T+evqu+Q==", + "dependencies": { + "buffer": "^6.0.3", + "inherits": "^2.0.4", + "readable-stream": "^4.2.0" + } + }, + "node_modules/tar-stream/node_modules/readable-stream": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.3.0.tgz", + "integrity": "sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ==", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -3769,16 +3898,16 @@ } }, "node_modules/typescript": { - "version": "4.9.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", - "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.2.tgz", + "integrity": "sha512-wVORMBGO/FAs/++blGNeAVdbNKtIh1rbBL2EyQ1+J9lClJ93KiiKe8PmFIVdXhHcyv44SL9oglmfeSsndo0jRw==", "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=4.2.0" + "node": ">=12.20" } }, "node_modules/unbzip2-stream": { diff --git a/package.json b/package.json index 31dd81f..8409eb7 100644 --- a/package.json +++ b/package.json @@ -36,27 +36,27 @@ "@types/express": "^4.17.17", "@types/inquirer": "^9.0.3", "@types/lzma-native": "^4.0.1", - "@types/node": "^18.14.2", + "@types/node": "^18.15.10", "@types/semver": "^7.3.13", "@types/tar": "^6.1.4", - "@types/yargs": "^17.0.22", + "@types/yargs": "^17.0.24", "ts-node": "^10.9.1", - "typescript": "^4.9.5" + "typescript": "^5.0.2" }, "dependencies": { - "@sirherobrine23/cloud": "^3.5.2", - "@sirherobrine23/debian": "^3.5.2", - "@sirherobrine23/docker-registry": "^3.5.2", - "@sirherobrine23/extends": "^3.5.2", - "@sirherobrine23/http": "^3.5.2", + "@sirherobrine23/cloud": "^3.5.4", + "@sirherobrine23/debian": "^3.5.4", + "@sirherobrine23/docker-registry": "^3.5.4", + "@sirherobrine23/extends": "^3.5.4", + "@sirherobrine23/http": "^3.5.4", "acme-client": "^5.0.0", "express": "^4.18.2", - "inquirer": "^9.1.4", + "inquirer": "^9.1.5", "lzma-native": "^8.0.6", "mongodb": "^5.1.0", "nano": "^10.1.2", "openpgp": "^5.7.0", - "ora": "^6.1.2", + "ora": "^6.3.0", "semver": "^7.3.8", "tar": "^6.1.13", "yaml": "^2.2.1", diff --git a/src/packageManege.ts b/src/packageManege.ts index b737495..e25c590 100644 --- a/src/packageManege.ts +++ b/src/packageManege.ts @@ -31,7 +31,7 @@ export async function fileRestore(packageDb: packageData, repoConfig: aptStreamC } else if (source.type === "docker") { const { image, auth } = source, { ref, path: debPath } = packageDb.fileRestore; const registry = new dockerRegistry(image, auth); - return new Promise((done, reject) => registry.extractLayer(ref).then(tar => tar.on("error", reject).on("File", entry => entry.path === debPath ? done(entry as any) : null))); + return new Promise((done, reject) => registry.extractLayer(ref).then(tar => tar.on("error", reject).on("File", entry => entry.path === debPath ? done(entry.stream) : null))); } throw new Error("Check package type"); } @@ -54,29 +54,33 @@ export class syncRepository extends EventEmitter { if (!source) continue; for (const target of source) { const { id } = target; - try { - if (target.type === "http") { - const { control } = await Debian.parsePackage(await coreHttp.streamRequest(target.url, {headers: target.auth?.header, query: target.auth?.query})); - this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)); - } else if (target.type === "oracle_bucket") { - const { authConfig, path = [] } = target; - const bucket = await oracleBucket.oracleBucket(authConfig); + if (target.type === "http") { + await coreHttp.streamRequest(target.url, {headers: target.auth?.header, query: target.auth?.query}) + .then(str => Debian.parsePackage(str) + .then(({control}) => packageManeger.addPackage(repo, target.componentName || "main", id, {}, control))) + .then(src => this.emit("addPackage", src)).catch(err => this.emit("error", err)); + } else if (target.type === "oracle_bucket") { + const { authConfig, path = [] } = target; + await oracleBucket.oracleBucket(authConfig).then(async bucket => { if (path.length === 0) path.push(...((await bucket.listFiles()).filter(k => k.name.endsWith(".deb")).map(({name}) => name))); - await Promise.all(path.map(async file => { + for (const file of path) { const { control } = await Debian.parsePackage(await bucket.getFileStream(file)); this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)); - })); - } else if (target.type === "google_driver") { - const { clientId, clientSecret, clientToken, gIds = [] } = target; - const gdrive = await googleDriver.GoogleDriver({clientID: clientId, clientSecret, token: clientToken}); + } + }).catch(err => this.emit("error", err)); + } else if (target.type === "google_driver") { + const { clientId, clientSecret, clientToken, gIds = [] } = target; + if (!clientToken) { + this.emit("error", new Error(`Cannot get files from ${id}, Google driver token is blank`)); + continue; + } + await googleDriver.GoogleDriver({clientID: clientId, clientSecret, token: clientToken}).then(async gdrive => { if (gIds.length === 0) gIds.push(...((await gdrive.listFiles()).filter(rel => rel.name.endsWith(".deb")).map(({id}) => id))); - await Promise.all(gIds.map(async file => { - const { control } = await Debian.parsePackage(await gdrive.getFileStream(file)); - this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)); - })); - } else if (target.type === "github") { - const { owner, repository, token } = target; - const gh = await Github.GithubManeger(owner, repository, token); + for (const file of gIds) await Debian.parsePackage(await gdrive.getFileStream(file)).then(({control}) => packageManeger.addPackage(repo, target.componentName || "main", id, {}, control)).then(src => this.emit("addPackage", src)); + }).catch(err => this.emit("error", err)); + } else if (target.type === "github") { + const { owner, repository, token } = target; + await Github.GithubManeger(owner, repository, token).then(async gh => { if (target.subType === "branch") { const { branch = (await gh.branchList()).at(0)?.name ?? "main" } = target; for (const { path: filePath } of (await gh.trees(branch)).tree.filter(file => file.type === "tree" ? false : file["path"])) { @@ -94,27 +98,33 @@ export class syncRepository extends EventEmitter { } } } - } else if (target.type === "docker") { - const { image, auth, tags = [] } = target; - const registry = new dockerRegistry(image, auth); + }).catch(err => this.emit("error", err)); + } else if (target.type === "docker") { + const { image, auth, tags = [] } = target; + const registry = new dockerRegistry(image, auth); + const userAuth = new dockerAuth(registry.image, "pull", auth); + try { if (tags.length === 0) { const { sha256, tag } = registry.image; if (sha256) tags.push(sha256); else if (tag) tags.push(tag); else tags.push(...((await registry.getTags()).reverse().slice(0, 6))); } - const userAuth = new dockerAuth(registry.image, "pull", auth); - await userAuth.setup(); + } catch (err) { + this.emit("error", err); + continue; + } + + await userAuth.setup().then(async () => { for (const tag of tags) { const manifestManeger = new dockerUtils.Manifest(await registry.getManifets(tag, userAuth), registry); const addPckage = async () => { for (const layer of manifestManeger.getLayers()) { - const blob = await registry.extractLayer(layer.digest, layer.mediaType, userAuth); + const blob = await registry.extractLayer(layer.digest, userAuth); blob.on("File", async entry => { if (!(entry.path.endsWith(".deb"))) return null; - console.log(entry.path); try { - const { control } = await Debian.parsePackage(entry as any); + const { control } = await Debian.parsePackage(entry.stream as any); this.emit("addPackage", await packageManeger.addPackage(repo, target.componentName || "main", id, { ref: layer.digest, path: entry.path, @@ -131,9 +141,7 @@ export class syncRepository extends EventEmitter { } } else await addPckage(); } - } - } catch (err) { - this.emit("error", err); + }).then(err => this.emit("error", err)); } } } -- 2.47.2 From e0efdbde3916c0e5c0599464fc52e19fc6953d03 Mon Sep 17 00:00:00 2001 From: Matheus Sampaio Queiroga Date: Tue, 28 Mar 2023 17:17:02 -0300 Subject: [PATCH 5/6] add package download --- src/aptServer.ts | 10 ++++++++++ src/packageManege.ts | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/aptServer.ts b/src/aptServer.ts index 0ffa846..14b28dc 100644 --- a/src/aptServer.ts +++ b/src/aptServer.ts @@ -2,6 +2,7 @@ import * as Debian from "@sirherobrine23/debian"; import { packageManeger, packageData } from "./database.js"; import { aptStreamConfig } from "./config.js"; import { extendsCrypto } from "@sirherobrine23/extends"; +import { fileRestore } from "./packageManege.js" import express from "express"; import stream from "node:stream"; import openpgp from "openpgp"; @@ -116,6 +117,7 @@ export default function main(packageManeger: packageManeger, config: aptStreamCo Filename: path.resolve("/", pathRoot ?? "", "pool", packageComponent ?? "main", `${control.Package}_${control.Architecture}_${control.Version}.deb`), }))); } + __stream.push(null); return extendsCrypto.createHashAsync(comp ? comp : __stream); }); return Object.assign(comp ? comp : __stream, __load); @@ -149,5 +151,13 @@ export default function main(packageManeger: packageManeger, config: aptStreamCo })); }); + app.get("/pool/:componentName/(:package)_(:arch)_(:version).deb", async (req, res, next) => { + const { componentName, package: packageName, arch, version: packageVersion } = req.params; + const packageID = (await packageManeger.search({packageComponent: componentName, packageArch: arch})).find(({packageControl: { Package, Version }}) => packageName === Package && Version === packageVersion); + if (!packageID) return res.status(404).json({error: "Package not exist"}); + console.log(packageID); + return fileRestore(packageID, config).then(str => str.pipe(res.writeHead(200, {}))).catch(next); + }); + return app; } \ No newline at end of file diff --git a/src/packageManege.ts b/src/packageManege.ts index e25c590..3f6d0cc 100644 --- a/src/packageManege.ts +++ b/src/packageManege.ts @@ -19,6 +19,7 @@ export async function fileRestore(packageDb: packageData, repoConfig: aptStreamC return coreHttp.streamRequest(url, {headers: header, query}); } else if (source.type === "github") { const { token } = source, { url } = packageDb.fileRestore; + return coreHttp.streamRequest(url, {headers: token ? {"Authorization": "token "+token} : {}}); } else if (source.type === "oracle_bucket") { const { authConfig } = source, { fileRestore: { path } } = packageDb; @@ -114,7 +115,7 @@ export class syncRepository extends EventEmitter { this.emit("error", err); continue; } - + await userAuth.setup().then(async () => { for (const tag of tags) { const manifestManeger = new dockerUtils.Manifest(await registry.getManifets(tag, userAuth), registry); -- 2.47.2 From 7304a1280f5a4d464d34e747273795bcfbdd6219 Mon Sep 17 00:00:00 2001 From: Matheus Sampaio Queiroga Date: Wed, 29 Mar 2023 13:18:54 -0300 Subject: [PATCH 6/6] To merge --- .github/workflows/nightly_build.yaml | 2 - .github/workflows/publish.yaml | 62 +++++++++---------- .gitignore | 1 + .npmignore | 12 +++- Dockerfile | 4 +- src/aptServer.ts | 8 ++- src/config.ts | 27 ++++++++- src/configManeger.ts | 89 +++++++++++++++++++++++++++- src/database.ts | 2 +- src/index.ts | 28 ++++++--- src/log.ts | 2 +- 11 files changed, 180 insertions(+), 57 deletions(-) diff --git a/.github/workflows/nightly_build.yaml b/.github/workflows/nightly_build.yaml index a71a29d..6ea01d2 100644 --- a/.github/workflows/nightly_build.yaml +++ b/.github/workflows/nightly_build.yaml @@ -28,9 +28,7 @@ jobs: - name: Build image uses: docker/build-push-action@v4 with: - cache-from: ${{ github.event_name == 'push' && 'type=gha,scope=${{ github.ref }}_${{ github.repo }}' || '' }} platforms: "linux/amd64,linux/arm64" - cache-to: type=gha,scope=${{ github.ref }}_${{ github.repo }} context: ./ push: true tags: ghcr.io/sirherobrine23/apt-stream:nightly diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 3087626..06734d5 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -9,6 +9,11 @@ jobs: publishpackage: runs-on: ubuntu-latest name: Publish + permissions: + packages: write + contents: write + env: + PACKAGE_VERSION: ${{ github.ref }} steps: - uses: actions/checkout@v3 name: Code checkout @@ -18,39 +23,6 @@ jobs: fetch-depth: 2 submodules: true - # Install basic tools - - uses: actions/setup-node@v3 - name: Setup node.js - with: - node-version: 18.x - registry-url: https://registry.npmjs.org/ - - - run: sudo npm install -g ts-node typescript - name: Install typescript and ts-node - - - name: Edit version - shell: node {0} - run: | - const fs = require("fs"); - const path = require("path"); - const packagePath = path.join(process.cwd(), "package.json"); - const package = JSON.parse(fs.readFileSync(packagePath, "utf8")); - package.version = "${{ github.ref }}"; - package.version = package.version.replace(/[A-Za-z_\/]+/, ""); - fs.writeFileSync(packagePath, JSON.stringify(package, null, 2)); - - # Add version to environment variables - - name: Add version to environment variables - run: | - cat package.json | jq -r '.version' > /tmp/version.txt - echo "PACKAGE_VERSION=$(cat /tmp/version.txt)" >> $GITHUB_ENV - - # Install depencides and build - - run: npm ci - - # Build - - run: npm run build - - name: Setup QEMU to Docker uses: docker/setup-qemu-action@v2 @@ -64,12 +36,32 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # Install basic tools + - uses: actions/setup-node@v3 + name: Setup node.js + with: + node-version: 18.x + registry-url: https://registry.npmjs.org/ + + - name: Edit version and install depencies + run: | + sudo npm i -g semver + VERSION="$(semver -c ${{ github.ref_name }})" + echo "PACKAGE_VERSION=$VERSION" >> $GITHUB_ENV + jq --arg ver $VERSION '.version = $ver' package.json + + # Install depencides and build + npm install --no-save + + # Publish npm + - run: npm publish --access public --tag ${{ github.event.release.prerelease && 'next' || 'latest' }} + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - name: Build image uses: docker/build-push-action@v4 with: - cache-from: ${{ github.event_name == 'push' && 'type=gha,scope=${{ github.ref }}_${{ github.repo }}' || '' }} platforms: "linux/amd64,linux/arm64" - cache-to: type=gha,scope=${{ github.ref }}_${{ github.repo }} context: ./ push: true tags: | diff --git a/.gitignore b/.gitignore index 097816d..3bf58ab 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ thunder-tests *.deb *.key *.pem +*.gpg # Ingore node folders node_modules/ diff --git a/.npmignore b/.npmignore index 2cae04c..52f66a0 100644 --- a/.npmignore +++ b/.npmignore @@ -16,12 +16,18 @@ node_modules/ # Docker .dockerignore -.Dockerfile +*docker-compose.yaml +*docker-compose.yml +*Dockerfile* +*dockerfile* # Project -/apt*.y[a]ml +/apt*.yml +/apt*.yaml +/apt*.json .repoTest/ thunder-tests *.deb *.key -*.pem \ No newline at end of file +*.pem +*.gpg \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6a3e349..5232f49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN npm run build # Clean build FROM node:lts VOLUME [ "/data" ] -COPY --from=0 /app/ /app WORKDIR /app +COPY --from=0 /app/ ./ RUN npm link -ENTRYPOINT "apt-stream server --cache /data/cache --data /data/data" \ No newline at end of file +ENTRYPOINT [ "apt-stream", "server", "--cache", "/data/cache" ] \ No newline at end of file diff --git a/src/aptServer.ts b/src/aptServer.ts index 14b28dc..f0389f7 100644 --- a/src/aptServer.ts +++ b/src/aptServer.ts @@ -150,7 +150,12 @@ export default function main(packageManeger: packageManeger, config: aptStreamCo return createPackage(packages, path.resolve("/", path.posix.join(req.baseUrl, req.path), "../../../../.."), req.path.endsWith(".gzip") ? "gzip" : req.path.endsWith(".xz") ? "lzma" : undefined).pipe(res.writeHead(200, { })); }); - + app.get("/pool", async ({res}) => packageManeger.search({}).then(data => res.json(data))); + app.get("/pool/:componentName", async (req, res) => { + const packagesList = await packageManeger.search({packageComponent: req.params.componentName}); + if (packagesList.length === 0) return res.status(404).json({error: "Package component not exists"}); + return res.json(packagesList.map(({packageControl, packageDistribuition}) => ({control: packageControl, dist: packageDistribuition}))); + }); app.get("/pool/:componentName/(:package)_(:arch)_(:version).deb", async (req, res, next) => { const { componentName, package: packageName, arch, version: packageVersion } = req.params; const packageID = (await packageManeger.search({packageComponent: componentName, packageArch: arch})).find(({packageControl: { Package, Version }}) => packageName === Package && Version === packageVersion); @@ -158,6 +163,5 @@ export default function main(packageManeger: packageManeger, config: aptStreamCo console.log(packageID); return fileRestore(packageID, config).then(str => str.pipe(res.writeHead(200, {}))).catch(next); }); - return app; } \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 85f652f..90ea0c2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -155,6 +155,11 @@ export async function prettyConfig(tmpConfig: aptStreamConfig, optionsOverload?: repository: {}, }; + if (newConfigObject.gpgSign) { + if (!newConfigObject.gpgSign.private.content && newConfigObject.gpgSign.private.path) newConfigObject.gpgSign.private.content = await fs.readFile(path.resolve(process.cwd(), newConfigObject.gpgSign.private.path), "utf8"); + if (!newConfigObject.gpgSign.public.content && newConfigObject.gpgSign.public.path) newConfigObject.gpgSign.public.content = await fs.readFile(path.resolve(process.cwd(), newConfigObject.gpgSign.public.path), "utf8"); + } + for (const repoName of returnUniq((Object.keys(optionsOverload?.repository ?? {}).concat(...(Object.keys(tmpConfig.repository ?? {})))))) { for (const data of ((optionsOverload?.repository?.[repoName]?.source ?? []).concat(tmpConfig?.repository?.[repoName]?.source)).filter(Boolean)) { if (!data) continue; @@ -228,10 +233,30 @@ export async function prettyConfig(tmpConfig: aptStreamConfig, optionsOverload?: return newConfigObject; } +export async function convertString(config: aptStreamConfig, target: "yaml"|"yml"|"json"|"json64"|"yaml64"|"yml64") { + config = await prettyConfig(config); + let encode64 = target.endsWith("64"); + let configString: string; + if (target === "json"||target === "json64") configString = JSON.stringify(config, null, encode64 ? 0 : 2); + else configString = yaml.stringify(config); + if (encode64) return Buffer.from(configString, "utf8").toString("base64"); + return configString; +} + export async function save(configPath: string, config: aptStreamConfig) { + config = await prettyConfig(config); + if (config.gpgSign) { + if (config.gpgSign.private.path) { + await fs.writeFile(path.resolve(process.cwd(), config.gpgSign.private.path), config.gpgSign.private.content); + config.gpgSign.private.content = null; + } + if (config.gpgSign.public.path) { + await fs.writeFile(path.resolve(process.cwd(), config.gpgSign.public.path), config.gpgSign.public.content); + config.gpgSign.public.content = null; + } + } let ext = ".json"; if (path.extname(configPath) === ".yaml" || path.extname(configPath) === ".yml") ext = ".yaml"; - config = await prettyConfig(config); return fs.writeFile(configPath, ext === ".json" ? JSON.stringify(config, null, 2) : yaml.stringify(config)); } diff --git a/src/configManeger.ts b/src/configManeger.ts index 5003d9f..d933011 100644 --- a/src/configManeger.ts +++ b/src/configManeger.ts @@ -5,7 +5,11 @@ import { googleDriver, oracleBucket } from "@sirherobrine23/cloud"; import { syncRepository } from "./packageManege.js"; import { connect } from "./database.js"; import { Github } from "@sirherobrine23/http"; +import openpgp from "openpgp"; import ora from "ora"; +import path from "path"; +import fs from "fs/promises"; +import { extendsFS } from "@sirherobrine23/extends"; async function simpleQuestion(promp: QuestionCollection): Promise> { promp["name"] ??= "No name"; @@ -343,6 +347,87 @@ async function manegerSource(config: aptStreamConfig, repositoryName: string): P return config; } +async function genGPG(config: aptStreamConfig): Promise { + if (config.gpgSign) console.warn("Replacing exists gpg keys"); + + const ask = await inquirer.prompt([ + { + type: "input", + message: "Full name or nickname, example Google Inc.:", + name: "name", + }, + { + type: "input", + message: "email, example: noreply@gmail.com:", + name: "email" + }, + { + type: "password", + mask: "*", + message: "password to encrypt the gpg files, if you don't want to leave it blank", + name: "pass", + validate(input = "") { + if (input.length === 0) return true; + else if (input.length >= 8) return true; + return "Password must have more than 8 characters!"; + }, + }, + { + type: "password", + mask: "*", + name: "passConfirm", + when: (answers) => answers.pass?.length > 0, + validate(input, answers) { + if (input === answers.pass) return true; + return "Invalid password, check is same!"; + }, + }, + { + type: "confirm", + message: "Want to save keys locally?", + name: "confirmSaveGPG", + }, + { + type: "input", + message: "Which folder do you save?", + name: "folderPath", + default: path.resolve(process.cwd(), "gpgKeys"), + when: (answers) => answers.confirmSaveGPG + } + ]); + return openpgp.generateKey({ + rsaBits: 4096, + format: "armored", + type: "rsa", + passphrase: ask.pass, + userIDs: [{ + comment: "Generated by apt-stream", + name: ask.name, + email: ask.email, + }], + }).then(async keys => { + config.gpgSign = { + authPassword: ask.pass, + private: { + content: keys.privateKey, + }, + public: { + content: keys.publicKey, + } + }; + if (ask.confirmSaveGPG) { + const folderPath = path.resolve(process.cwd(), ask.folderPath); + if (!(await extendsFS.exists(folderPath))) await fs.mkdir(folderPath, {recursive: true}); + config.gpgSign.private.path = path.join(folderPath, "privateAptStream.gpg"); + config.gpgSign.public.path = path.join(folderPath, "publicAptStream.gpg"); + } + return config; + }).catch(err => { + console.error(err?.message || err); + return genGPG(config); + }); +} + export default async function main(configPath: string, configOld?: aptStreamConfig) { if (configOld) { console.log("Saving current config..."); @@ -353,18 +438,20 @@ export default async function main(configPath: string, configOld?: aptStreamConf console.log("Init fist repository config!"); return createRepository(localConfig).then(d => main(configPath, d)); } - const target = await simpleQuestion<"new"|"edit"|"load"|"exit">({ + const target = await simpleQuestion<"new"|"gpg"|"edit"|"load"|"exit">({ type: "list", message: "Select action", choices: [ {name: "Edit repository", value: "edit"}, {name: "Create new Repository", value: "new"}, + {name: "(Re)generate gpg keys", value: "gpg"}, {name: "Sync repository", value: "load"}, {name: "Exit", value: "exit"} ] }); if (target !== "exit") { if (target === "new") configOld = await createRepository(localConfig); + if (target === "gpg") configOld = await genGPG(localConfig); else if (target === "edit") { const repoName = await simpleQuestion({ type: "list", diff --git a/src/database.ts b/src/database.ts index 475c6a7..a4398cf 100644 --- a/src/database.ts +++ b/src/database.ts @@ -24,7 +24,7 @@ export class packageManeger { return this.options.getPackages.call(this); } - search = async (search: {packageName?: string, packageArch?: string, packageComponent?: string, packageDist?: string}): ReturnType => { + async search(search: {packageName?: string, packageArch?: string, packageComponent?: string, packageDist?: string}): Promise { if (typeof this.options.findPackages !== "function") return (await this.getPackages()).filter(data => ((!search.packageName) || (search.packageName === data.packageControl.Package)) && ((!search.packageArch) || (data.packageControl.Architecture === search.packageArch)) && ((!search.packageComponent) || (data.packageComponent === search.packageComponent)) && ((!search.packageDist) || (data.packageDistribuition === search.packageDist))); return this.options.findPackages.call(this, search); } diff --git a/src/index.ts b/src/index.ts index eb9cbe2..32035e0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,15 +1,15 @@ #!/usr/bin/env node import "./log.js"; import { connect } from "./database.js"; -import { config } from "./config.js"; +import { config, convertString } from "./config.js"; import packageManeger from "./configManeger.js"; import express from "express"; import yargs from "yargs"; import cluster from "node:cluster"; import apt from "./aptServer.js"; +import openpgp from "openpgp"; -yargs(process.argv.slice(2)).version(false).help(true).strictCommands().demandCommand().alias("h", "help").command("server", "Run http Server", async yargs => { - const options = yargs.option("config", { +yargs(process.argv.slice(2)).version(false).help(true).strictCommands().demandCommand().alias("h", "help").command("server", "Run http Server", yargs => yargs.option("config", { string: true, alias: "c", type: "string", @@ -30,7 +30,7 @@ yargs(process.argv.slice(2)).version(false).help(true).strictCommands().demandCo alias: "C", type: "string", description: "cache files" - }).parseSync(); + }), async options => { const appConfig = await config(options.config, {serverConfig: {portListen: options.port, clusterCount: options.cluster, cacheFolder: options.cache}}); if ((appConfig.serverConfig?.clusterCount || 0) > 0 && cluster.isPrimary) { const ct = () => { @@ -47,19 +47,29 @@ yargs(process.argv.slice(2)).version(false).help(true).strictCommands().demandCo } const db = await connect(appConfig); const app = express(); + app.get("/", ({res}) => res.json({cluster: cluster.isWorker, id: cluster.worker?.id})); + app.get("/public(_key|)(|.gpg)", async ({res}) => { + if (!appConfig.gpgSign) return res.status(404).json({error: "Gpg not configured"}); + const pubKey = (await openpgp.readKey({ armoredKey: appConfig.gpgSign.public.content })).armor(); + return res.setHeader("Content-Type", "application/pgp-keys").send(pubKey); + }); const aptRoute = apt(db, appConfig); app.use(aptRoute); app.listen(appConfig.serverConfig?.portListen ?? 0, function () { const address = this.address(); console.log("Port Listen on %O", typeof address === "object" ? address.port : address); }); -}).command("package", "maneger packages in database", yargs => { - const { config } = yargs.option("config", { +}).command(["maneger", "m", "$0"], "maneger packages in database", yargs => { + return yargs.option("config", { string: true, alias: "c", type: "string", description: "Config file path", - default: "aptStream.yml" - }).parseSync(); - return packageManeger(config); + default: "aptStream.yml", + }).command(["$0"], "Maneger config", yargs => yargs, options => packageManeger(options.config)).command(["print", "p"], "Print config to target default is json", yargs => yargs.option("outputType", { + alias: "o", + choices: ["yaml", "yml", "json", "json64", "yaml64", "yml64"], + description: "target output file, targets ended with '64' is base64 string", + default: "json" + }), async (options) => console.log(await convertString(await config(options.config), options.outputType as any))); }).parseAsync(); \ No newline at end of file diff --git a/src/log.ts b/src/log.ts index ccb5ed5..a9b77d0 100644 --- a/src/log.ts +++ b/src/log.ts @@ -10,7 +10,7 @@ if (cluster.isWorker) { depth: null }; - console.clear = console.clear ?? function () {console.warn("Not tty")} + console.clear = console.clear ?? function () {console.warn("cannot clear tty");} console.log = function(...args) { log("[LOG%s]: %s", id ? ` Cluster ${id}` : "", formatWithOptions(defaultOptions, ...args)); -- 2.47.2