diff --git a/.gitignore b/.gitignore index 035dd75..528656e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ tests/**/out out/ vendor/ tests/**/rpmoci.lock +__pycache__ diff --git a/Cargo.lock b/Cargo.lock index 8607bd3..4659766 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,9 +101,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "autocfg" @@ -167,9 +167,9 @@ checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" [[package]] name = "cap-primitives" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d00bd8d26c4270d950eaaa837387964a2089a1c3c349a690a1fa03221d29531" +checksum = "ff5bcbaf57897c8f14098cc9ad48a78052930a9948119eea01b80ca224070fa6" dependencies = [ "ambient-authority", "fs-set-times", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "cap-std" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19eb8e3d71996828751c1ed3908a439639752ac6bdc874e41469ef7fc15fbd7f" +checksum = "e6cf1a22e6eab501e025a9953532b1e95efb8a18d6364bf8a4a7547b30c49186" dependencies = [ "cap-primitives", "io-extras", @@ -207,9 +207,9 @@ dependencies = [ [[package]] name = "cap-tempfile" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53880047c3f37cd64947775f0526795498d614182603a718c792616b762ce777" +checksum = "8563f37bd2d9ec79a08dc6b062b6733adc84f929d23f45388ba52025c7b32e26" dependencies = [ "cap-std", "rand", @@ -219,9 +219,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.13" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "jobserver", "libc", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -276,9 +276,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -288,14 +288,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -330,9 +330,9 @@ checksum = "80e3adec7390c7643049466136117057188edf5f23efc5c8b4fc8079c8dc34a6" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -377,7 +377,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -388,38 +388,38 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] name = "derive_builder" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" +checksum = "cd33f37ee6a119146a1781d3356a7c26028f83d779b2e04ecd45fdc75c76877b" dependencies = [ "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" +checksum = "7431fa049613920234f22c47fdc33e6cf3ee83067091ea4277a3f8c4587aae38" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] name = "derive_builder_macro" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" +checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc" dependencies = [ "derive_builder_core", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -432,7 +432,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -484,7 +484,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -495,7 +495,7 @@ checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" dependencies = [ "num-traits", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -551,15 +551,15 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "filetime" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", @@ -569,26 +569,15 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", "libz-sys", "miniz_oxide", ] -[[package]] -name = "fn-error-context" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd66269887534af4b0c3e3337404591daa8dc8b9b2b3db71f9523beb4bafb41" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.75", -] - [[package]] name = "fnv" version = "1.0.7" @@ -653,14 +642,14 @@ dependencies = [ [[package]] name = "getset" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" +checksum = "f636605b743120a8d32ed92fc27b6cde1a769f8f936c065151eb66f88ded513c" dependencies = [ - "proc-macro-error", + "proc-macro-error2", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.77", ] [[package]] @@ -693,6 +682,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "hex" version = "0.4.3" @@ -707,9 +702,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -746,9 +741,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown", @@ -778,9 +773,9 @@ checksum = "5a611371471e98973dbcab4e0ec66c31a10bc356eeb4d54a0e05eac8158fe38c" [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "is_terminal_polyfill" @@ -823,9 +818,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libredox" @@ -850,9 +845,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.19" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc53a7799a7496ebc9fd29f31f7df80e83c9bda5299768af5f9e59eeea74647" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "pkg-config", @@ -991,7 +986,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -1034,15 +1029,24 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "oci-spec" -version = "0.6.8" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f5a3fe998d50101ae009351fec56d88a69f4ed182e11000e711068c2f5abf72" +checksum = "5cee185ce7cf1cce45e194e34cd87c0bad7ff0aa2e8917009a2da4f7b31fb363" dependencies = [ "derive_builder", "getset", - "once_cell", "regex", "serde", "serde_json", @@ -1053,16 +1057,13 @@ dependencies = [ [[package]] name = "ocidir" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d10ccb21fb07204f7177e9b44dfb5bafc8a7962f980ede68d60084ca658c9bc7" +version = "0.3.1" +source = "git+https://github.com/tofay/ocidir-rs?branch=layer-media-type#bb59a9902147ae9eb39e9f5dda7cb95acb7df2cd" dependencies = [ - "anyhow", "camino", "cap-std-ext", "chrono", "flate2", - "fn-error-context", "hex", "oci-spec", "olpc-cjson", @@ -1070,6 +1071,7 @@ dependencies = [ "serde", "serde_json", "tar", + "thiserror", ] [[package]] @@ -1112,7 +1114,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -1141,15 +1143,15 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" [[package]] name = "ppv-lite86" @@ -1161,27 +1163,25 @@ dependencies = [ ] [[package]] -name = "proc-macro-error" -version = "1.0.4" +name = "proc-macro-error-attr2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" dependencies = [ - "proc-macro-error-attr", "proc-macro2", "quote", - "syn 1.0.109", - "version_check", ] [[package]] -name = "proc-macro-error-attr" -version = "1.0.4" +name = "proc-macro-error2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" dependencies = [ + "proc-macro-error-attr2", "proc-macro2", "quote", - "version_check", + "syn 2.0.77", ] [[package]] @@ -1195,9 +1195,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831e8e819a138c36e212f3af3fd9eeffed6bf1510a805af35b0edee5ffa59433" +checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225" dependencies = [ "cfg-if", "indoc", @@ -1213,9 +1213,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e8730e591b14492a8945cdff32f089250b05f5accecf74aeddf9e8272ce1fa8" +checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3" dependencies = [ "once_cell", "target-lexicon", @@ -1223,9 +1223,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e97e919d2df92eb88ca80a037969f44e5e70356559654962cbb3316d00300c6" +checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c" dependencies = [ "libc", "pyo3-build-config", @@ -1233,34 +1233,34 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb57983022ad41f9e683a599f2fd13c3664d7063a3ac5714cae4b7bee7d3f206" +checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] name = "pyo3-macros-backend" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec480c0c51ddec81019531705acac51bcdbeae563557c982aa8263bb96880372" +checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1" dependencies = [ "heck", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -1297,9 +1297,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0" dependencies = [ "bitflags", ] @@ -1369,13 +1369,16 @@ dependencies = [ "chrono", "clap", "clap-verbosity-flag", + "derive_builder", "env_logger", "filetime", "flate2", "glob", "log", "nix", + "num_cpus", "ocidir", + "openssl", "pathdiff", "pyo3", "rpm", @@ -1390,6 +1393,7 @@ dependencies = [ "url", "walkdir", "xattr", + "zstd", ] [[package]] @@ -1408,18 +1412,18 @@ dependencies = [ [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -1459,29 +1463,29 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.208" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.208" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] name = "serde_json" -version = "1.0.125" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", "memchr", @@ -1554,7 +1558,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -1570,9 +1574,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.75" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -1581,9 +1585,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" dependencies = [ "filetime", "libc", @@ -1632,22 +1636,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] @@ -1688,9 +1692,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", "serde", @@ -1713,15 +1717,15 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -1809,7 +1813,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", "wasm-bindgen-shared", ] @@ -1831,7 +1835,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1944,9 +1948,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587" dependencies = [ "memchr", ] @@ -1999,7 +2003,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.77", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 26218a4..67aa324 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,11 @@ toml = { version = "0.8.8" } url = { version = "2.2.2", features = ["serde"] } walkdir = "2.3.2" xattr = "1.0.1" -ocidir = "0.2.1" +ocidir = { git = "https://github.com/tofay/ocidir-rs", branch = "layer-media-type" } +derive_builder = "0.20.1" +openssl = "0.10.66" +zstd = { version = "0.13.2", features = ["zstdmt"] } +num_cpus = "1.16.0" [dev-dependencies] test-temp-dir = "0.2.2" @@ -58,3 +62,7 @@ dnf = "*" default = ["test-docker"] # The "test-docker" feature is used to run integration tests requiring skopeo and docker test-docker = [] + +[[bench]] +name = "build_benchmark" +harness = false diff --git a/src/archive.rs b/src/archive.rs deleted file mode 100644 index e27795f..0000000 --- a/src/archive.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Copyright (C) Microsoft Corporation. -//! -//! This program is free software: you can redistribute it and/or modify -//! it under the terms of the GNU General Public License as published by -//! the Free Software Foundation, either version 3 of the License, or -//! (at your option) any later version. -//! -//! This program is distributed in the hope that it will be useful, -//! but WITHOUT ANY WARRANTY; without even the implied warranty of -//! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//! GNU General Public License for more details. -//! -//! You should have received a copy of the GNU General Public License -//! along with this program. If not, see . -use anyhow::{Context, Result}; -use std::{ - collections::{hash_map::Entry, HashMap}, - io::Write, - os::unix::{ - fs::MetadataExt, - prelude::{FileTypeExt, OsStrExt}, - }, - path::{Path, PathBuf}, -}; -use walkdir::WalkDir; - -// https://mgorny.pl/articles/portability-of-tar-features.html#id25 -const PAX_SCHILY_XATTR: &[u8; 13] = b"SCHILY.xattr."; - -/// custom implementation of tar-rs's append_dir_all that: -/// - works around https://github.com/alexcrichton/tar-rs/issues/102 so that security capabilities are preserved -/// - emulates tar's `--clamp-mtime` option so that any file/dir/symlink mtimes are no later than a specific value -/// - supports hardlinks -pub(super) fn append_dir_all_with_xattrs( - builder: &mut tar::Builder, - src_path: impl AsRef, - clamp_mtime: i64, -) -> Result<()> { - let src_path = src_path.as_ref(); - // Map (dev, inode) -> path for hardlinks - let mut hardlinks: HashMap<(u64, u64), PathBuf> = HashMap::new(); - - for entry in WalkDir::new(src_path) - .follow_links(false) - .sort_by_file_name() - .into_iter() - { - let entry = entry?; - let meta = entry.metadata()?; - // skip sockets as tar-rs errors when trying to archive them. - // For comparison, umoci also errors, whereas docker skips them - if meta.file_type().is_socket() { - continue; - } - - let rel_path = pathdiff::diff_paths(entry.path(), src_path) - .expect("walkdir returns path inside of search root"); - if rel_path == Path::new("") { - continue; - } - - if entry.file_type().is_symlink() { - if meta.mtime() > clamp_mtime { - // Setting the mtime on a symlink is fiddly with tar-rs, so we use filetime to change - // the mtime before adding the symlink to the tar archive - let mtime = filetime::FileTime::from_unix_time(clamp_mtime, 0); - filetime::set_symlink_file_times(entry.path(), mtime, mtime)?; - } - add_pax_extension_header(entry.path(), builder)?; - builder.append_path_with_name(entry.path(), rel_path)?; - } else if entry.file_type().is_file() || entry.file_type().is_dir() { - add_pax_extension_header(entry.path(), builder)?; - - // If this is a hardlink, add a link header instead of the file - // if this isn't the first time we've seen this inode - if meta.nlink() > 1 { - match hardlinks.entry((meta.dev(), meta.ino())) { - Entry::Occupied(e) => { - // Add link header and continue to next entry - let mut header = tar::Header::new_gnu(); - header.set_metadata(&meta); - if meta.mtime() > clamp_mtime { - header.set_mtime(clamp_mtime as u64); - } - header.set_entry_type(tar::EntryType::Link); - header.set_cksum(); - builder.append_link(&mut header, &rel_path, e.get())?; - continue; - } - Entry::Vacant(e) => { - // This is the first time we've seen this inode - e.insert(rel_path.clone()); - } - } - } - - let mut header = tar::Header::new_gnu(); - header.set_size(meta.len()); - header.set_metadata(&meta); - if meta.mtime() > clamp_mtime { - header.set_mtime(clamp_mtime as u64); - } - if entry.file_type().is_file() { - builder.append_data( - &mut header, - rel_path, - &mut std::fs::File::open(entry.path())?, - )?; - } else { - builder.append_data(&mut header, rel_path, &mut std::io::empty())?; - }; - } - } - - Ok(()) -} - -// Convert any extended attributes on the specified path to a tar PAX extension header, and add it to the tar archive -fn add_pax_extension_header( - path: impl AsRef, - builder: &mut tar::Builder, -) -> Result<(), anyhow::Error> { - let path = path.as_ref(); - let xattrs = xattr::list(path) - .with_context(|| format!("Failed to list xattrs from `{}`", path.display()))?; - let mut pax_header = tar::Header::new_gnu(); - let mut pax_data = Vec::new(); - for key in xattrs { - let value = xattr::get(path, &key) - .with_context(|| { - format!( - "Failed to get xattr `{}` from `{}`", - key.to_string_lossy(), - path.display() - ) - })? - .unwrap_or_default(); - - // each entry is " =\n": https://www.ibm.com/docs/en/zos/2.3.0?topic=SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/paxex.html - let data_len = PAX_SCHILY_XATTR.len() + key.as_bytes().len() + value.len() + 3; - // Calculate the total length, including the length of the length field - let mut len_len = 1; - while data_len + len_len >= 10usize.pow(len_len.try_into().unwrap()) { - len_len += 1; - } - write!(pax_data, "{} ", data_len + len_len)?; - pax_data.write_all(PAX_SCHILY_XATTR)?; - pax_data.write_all(key.as_bytes())?; - pax_data.write_all("=".as_bytes())?; - pax_data.write_all(&value)?; - pax_data.write_all("\n".as_bytes())?; - } - if !pax_data.is_empty() { - pax_header.set_size(pax_data.len() as u64); - pax_header.set_entry_type(tar::EntryType::XHeader); - pax_header.set_cksum(); - builder.append(&pax_header, &*pax_data)?; - } - Ok(()) -} diff --git a/src/imager/archive.rs b/src/imager/archive.rs new file mode 100644 index 0000000..e38db7e --- /dev/null +++ b/src/imager/archive.rs @@ -0,0 +1,63 @@ +//! Copyright (C) Microsoft Corporation. +//! +//! This program is free software: you can redistribute it and/or modify +//! it under the terms of the GNU General Public License as published by +//! the Free Software Foundation, either version 3 of the License, or +//! (at your option) any later version. +//! +//! This program is distributed in the hope that it will be useful, +//! but WITHOUT ANY WARRANTY; without even the implied warranty of +//! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//! GNU General Public License for more details. +//! +//! You should have received a copy of the GNU General Public License +//! along with this program. If not, see . +use anyhow::{Context, Result}; +use std::{io::Write, os::unix::prelude::OsStrExt, path::Path}; + +// https://mgorny.pl/articles/portability-of-tar-features.html#id25 +const PAX_SCHILY_XATTR: &[u8; 13] = b"SCHILY.xattr."; + +// Convert any extended attributes on the specified path to a tar PAX extension header, and add it to the tar archive +pub(crate) fn add_pax_extension_header( + path: impl AsRef, + builder: &mut tar::Builder, +) -> Result<(), anyhow::Error> { + let path = path.as_ref(); + let xattrs = xattr::list(path) + .with_context(|| format!("Failed to list xattrs from `{}`", path.display()))?; + let mut pax_header = tar::Header::new_gnu(); + let mut pax_data = Vec::new(); + for key in xattrs { + let value = xattr::get(path, &key) + .with_context(|| { + format!( + "Failed to get xattr `{}` from `{}`", + key.to_string_lossy(), + path.display() + ) + })? + .unwrap_or_default(); + + // each entry is " =\n": https://www.ibm.com/docs/en/zos/2.3.0?topic=SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/paxex.html + let data_len = PAX_SCHILY_XATTR.len() + key.as_bytes().len() + value.len() + 3; + // Calculate the total length, including the length of the length field + let mut len_len = 1; + while data_len + len_len >= 10usize.pow(len_len.try_into().unwrap()) { + len_len += 1; + } + write!(pax_data, "{} ", data_len + len_len)?; + pax_data.write_all(PAX_SCHILY_XATTR)?; + pax_data.write_all(key.as_bytes())?; + pax_data.write_all("=".as_bytes())?; + pax_data.write_all(&value)?; + pax_data.write_all("\n".as_bytes())?; + } + if !pax_data.is_empty() { + pax_header.set_size(pax_data.len() as u64); + pax_header.set_entry_type(tar::EntryType::XHeader); + pax_header.set_cksum(); + builder.append(&pax_header, &*pax_data)?; + } + Ok(()) +} diff --git a/src/imager/graph.py b/src/imager/graph.py new file mode 100644 index 0000000..0abe967 --- /dev/null +++ b/src/imager/graph.py @@ -0,0 +1,94 @@ +# Copyright (C) Microsoft Corporation. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +import dnf +from collections import defaultdict +from nix_closure_graph import make_graph_segment_from_root, graph_popularity_contest + + +def create_package_graph(root): + """ + Create a graph of installed packages and their dependencies. + """ + conf = dnf.conf.Conf() + cache_dir = os.environ.get("RPMOCI_CACHE_DIR", "") + if cache_dir: + conf.cachedir = cache_dir + conf.logdir = cache_dir + + conf.installroot = root + base = dnf.Base(conf) + base.fill_sack() + query = dnf.query.Query(base.sack) + installed = query.installed() + graph = defaultdict(set) + + for pkg in installed: + for req in pkg.requires: + providers = installed.filter(provides=req) + if providers: + for provider in providers: + if pkg.name != provider.name and pkg not in graph[provider]: + graph[pkg].add(provider) + return graph + + +def remove_cycles(graph): + """ + Repeatedly remove cycles from a graph until it's a DAG. + """ + from graphlib import TopologicalSorter, CycleError + + while True: + try: + _order = [*TopologicalSorter(graph).static_order()] + break + except CycleError as e: + # Remove a cycle + graph[e.args[1][1]].remove(e.args[1][0]) + return graph + + +def most_popular_packages(root, n, size_threshold): + """ + Return the n most popular packages in the specified installroot + with a size greater than or equal to the specified threshold. + + """ + lookup = remove_cycles(create_package_graph(root)) + new_graph = {} + for pkg in lookup.keys(): + if pkg in new_graph: + continue + new_graph[pkg] = make_graph_segment_from_root(pkg, lookup) + + most_popular = [ + p + for (p, _count) in sorted( + graph_popularity_contest(new_graph).items(), + key=lambda x: (x[1], x[0]), + reverse=True, + ) + if p.size >= size_threshold + ] + return most_popular[:n] if len(most_popular) >= n else most_popular + + +# For testing via `python3 src/lockfile/graph.py` +if __name__ == "__main__": + pkgs = most_popular_packages("/", 100, 5 * 1024 * 1024) + import pdb + + pdb.set_trace() diff --git a/src/imager/layer.rs b/src/imager/layer.rs new file mode 100644 index 0000000..d8f7fbe --- /dev/null +++ b/src/imager/layer.rs @@ -0,0 +1,95 @@ +use std::{io::Write, str::FromStr as _}; + +use anyhow::Result; +use ocidir::{ + oci_spec::image::{MediaType, Sha256Digest}, + BlobWriter, GzipLayerWriter, Layer, OciDir, +}; +use zstd::Encoder; + +use super::{sha256_writer::Sha256Writer, CompressionAlgorithm}; + +pub(super) enum LayerWriter<'a> { + Gzip(GzipLayerWriter<'a>), + Zstd(ZstdLayerWriter<'a>), +} + +impl<'a> LayerWriter<'a> { + pub fn new( + ocidir: &'a OciDir, + compression_algorithm: CompressionAlgorithm, + compression_level: Option, + ) -> Result { + Ok(match compression_algorithm { + CompressionAlgorithm::Gzip => Self::Gzip(ocidir.create_gzip_layer( + compression_level.map(|l| flate2::Compression::new(l.try_into().unwrap())), + )?), + CompressionAlgorithm::Zstd => { + Self::Zstd(ZstdLayerWriter::new(ocidir, compression_level)?) + } + }) + } + + pub fn complete(self) -> Result { + match self { + Self::Gzip(writer) => Ok(writer.complete()?), + Self::Zstd(writer) => writer.complete(), + } + } +} + +impl<'a> Write for LayerWriter<'a> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + Self::Gzip(writer) => writer.write(buf), + Self::Zstd(writer) => writer.write(buf), + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match self { + Self::Gzip(writer) => writer.flush(), + Self::Zstd(writer) => writer.flush(), + } + } +} + +/// A writer for a zstd compressed layer. +pub struct ZstdLayerWriter<'a>(Sha256Writer>>); + +impl<'a> ZstdLayerWriter<'a> { + /// Create new zstd layer. + fn new(ocidir: &'a OciDir, compression: Option) -> Result { + let bw = ocidir.create_blob()?; + let mut encoder = Encoder::new(bw, compression.unwrap_or(0))?; + // Set the number of workers to the number of CPUs + // when in multi-threaded mode each zstd version is reproducible regardless of the number of threads + let num_workers = num_cpus::get(); + encoder.set_parameter(zstd::zstd_safe::CParameter::NbWorkers( + num_workers.try_into()?, + ))?; + Ok(Self(Sha256Writer::new(encoder))) + } + + /// Finish writing the layer. + pub fn complete(self) -> Result { + let (digest, encoder) = self.0.finish(); + let uncompressed_sha256 = Sha256Digest::from_str(&digest)?; + let blob = encoder.finish()?.complete()?; + Ok(Layer { + uncompressed_sha256, + blob, + media_type: MediaType::ImageLayerZstd, + }) + } +} + +impl<'a> Write for ZstdLayerWriter<'a> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.0.flush() + } +} diff --git a/src/imager/mod.rs b/src/imager/mod.rs new file mode 100644 index 0000000..3375755 --- /dev/null +++ b/src/imager/mod.rs @@ -0,0 +1,445 @@ +//! Module for building layered OCI images +//! +//! Copyright (C) Microsoft Corporation. +//! +//! This program is free software: you can redistribute it and/or modify +//! it under the terms of the GNU General Public License as published by +//! the Free Software Foundation, either version 3 of the License, or +//! (at your option) any later version. +//! +//! This program is distributed in the hope that it will be useful, +//! but WITHOUT ANY WARRANTY; without even the implied warranty of +//! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//! GNU General Public License for more details. +//! +//! You should have received a copy of the GNU General Public License +//! along with this program. If not, see . +use crate::write; +use anyhow::{Context, Result}; +use archive::add_pax_extension_header; +use chrono::DateTime; +use derive_builder::{Builder, UninitializedFieldError}; +use layer::LayerWriter; +use ocidir::cap_std::fs::Dir; +use ocidir::oci_spec::image::{Descriptor, MediaType}; +use ocidir::{new_empty_manifest, Layer, OciDir}; +use pyo3::types::{PyAnyMethods, PyModule, PyTuple}; +use pyo3::{FromPyObject, Python, ToPyObject}; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::fs; +use std::os::unix::fs::{FileTypeExt, MetadataExt}; +use std::path::{Path, PathBuf}; +use walkdir::WalkDir; + +mod archive; +mod layer; +mod sha256_writer; + +const CREATED_BY: &str = "Created by rpmoci"; + +#[derive(Debug, Builder)] +#[builder( + custom_constructor, + create_empty = empty, + build_fn(private, name = "fallible_build", error = "DummyError"), + pattern = "owned" +)] +/// An OCI image builder that can put RPMs into individual layers. +pub struct Imager { + /// The installroot + #[builder(setter(custom))] + filesystem_root: PathBuf, + /// The OCI directory where the image is being built + #[builder(setter(custom))] + oci_dir: OciDir, + /// The maximum number of layers to create. + /// The default is 125. + #[builder(default = "default_max_layers()")] + max_layers: usize, + /// The time the image was created. + /// If not set, the current time is used. + #[builder(default = "default_creation_time()")] + creation_time: DateTime, + /// The compression algorithm to use for the image layers. + #[builder(default)] + compression_algorithm: CompressionAlgorithm, + /// The compression level to use for the image layers. + /// + /// The default for zstd is 3, and the default for gzip is 6. + #[builder(default)] + compression_level: Option, + /// The OCI image configuration. + #[builder(default)] + config: ocidir::oci_spec::image::ImageConfiguration, + /// The OCI image manifest. + /// + /// The default is an empty manifest with a media type of "application/vnd.oci.image.manifest.v1+json" + #[builder(default = "default_manifest()")] + manifest: ocidir::oci_spec::image::ImageManifest, + /// The image tag + #[builder(default = "String::from(\"latest\")", setter(into))] + tag: String, + /// Minimum size threshold for packages. + /// If a package is below this size, it won't be in its own layer + /// + /// The default is 5MB + #[builder(default = "5 * 1024 * 1024")] + rpm_size_threshold: u64, +} + +/// The compression algorithm to use for the image layers. +#[derive(Debug, Default, Clone, Copy)] +pub enum CompressionAlgorithm { + /// Gzip compression + #[default] + Gzip, + /// Zstandard compression + Zstd, +} + +#[derive(Debug)] +struct DummyError; + +impl From for DummyError { + fn from(_ufe: UninitializedFieldError) -> DummyError { + DummyError + } +} + +impl ImagerBuilder { + /// Create a new builder with the given paths. + pub fn build(self) -> Imager { + self.fallible_build().expect("All fields are initialized") + } +} + +fn default_max_layers() -> usize { + 125 +} + +fn default_creation_time() -> DateTime { + chrono::Utc::now() +} + +fn default_manifest() -> ocidir::oci_spec::image::ImageManifest { + new_empty_manifest() + .media_type(MediaType::ImageManifest) + .build() + .unwrap() +} + +impl Imager { + /// Create a new builder with the given paths. + /// + /// The OCI directory will be created if it does not exist. + /// + /// Errors if the OCI directory cannot be created, or is a non-empty directory + /// that does not contain an OCI image. + pub fn with_paths( + filesystem_root: impl AsRef, + oci_dir: impl AsRef, + ) -> Result { + let filesystem_root = std::path::absolute(filesystem_root)?; + let oci_dir = oci_dir.as_ref(); + fs::create_dir_all(oci_dir).context(format!( + "Failed to create OCI image directory `{}`", + oci_dir.display() + ))?; + let dir = Dir::open_ambient_dir(oci_dir, ocidir::cap_std::ambient_authority()) + .context("Failed to open image directory")?; + let oci_dir = OciDir::ensure(&dir)?; + + Ok(ImagerBuilder { + filesystem_root: Some(filesystem_root), + oci_dir: Some(oci_dir), + ..ImagerBuilder::empty() + }) + } + + /// Build the OCI image, by walking the filesystem and creating layers for each package. + /// + /// Returns the descriptor for the image manifest. + pub fn create_image(self) -> Result { + // Determine most popular packages + let popular_packages = self.most_popular_packages()?; + // Create a a layer for each package + let mut package_layers = self.package_layers(&popular_packages)?; + let path_to_layer_map = path_to_layer_map(popular_packages); + // Create a catchall layer for any files not in the most popular package layers + let mut catchall = self.create_layer(CREATED_BY, self.creation_time.timestamp())?; + + // Walk the filesystem and add files to the appropriate layers + self.walk_filesystem(path_to_layer_map, &mut package_layers, &mut catchall)?; + // Finalize the image by writing the layers to the OCI image directory + self.finish(package_layers, catchall) + } + + fn package_layers<'a>(&'a self, py_pkgs: &[PyPackage]) -> Result>> { + // Create a layer for each package + py_pkgs + .iter() + .map(|py_pkg| { + self.create_layer( + format!( + "{} for package {}-{}.{}", + CREATED_BY, py_pkg.name, py_pkg.evr, py_pkg.arch + ), + py_pkg.buildtime, + ) + }) + .collect::>>() + } + + /// variation of tar-rs's append_dir_all that: + /// - works around https://github.com/alexcrichton/tar-rs/issues/102 so that security capabilities are preserved + /// - emulates tar's `--clamp-mtime` option so that any file/dir/symlink mtimes are no later than a specific value + /// - supports hardlinks + /// - adds files to the correct archive layer + fn walk_filesystem<'a>( + &self, + path_to_layer_map: HashMap, + package_layers: &mut [LayerBuilder<'a>], + catchall: &mut LayerBuilder<'a>, + ) -> Result<()> { + // Map (dev, inode) -> path for hardlinks + let mut hardlinks: HashMap<(u64, u64), PathBuf> = HashMap::new(); + + for entry in WalkDir::new(&self.filesystem_root) + .follow_links(false) + .sort_by_file_name() + .into_iter() + { + let entry = entry?; + let meta = entry.metadata()?; + // skip sockets as tar-rs errors when trying to archive them. + // For comparison, umoci also errors, whereas docker skips them + if meta.file_type().is_socket() { + continue; + } + + let rel_path = pathdiff::diff_paths(entry.path(), &self.filesystem_root) + .expect("walkdir returns path inside of search root"); + if rel_path == Path::new("") { + continue; + } + + // Determine which builder to use + let wrapped_builder = match path_to_layer_map.get(&rel_path) { + Some(i) => &mut package_layers[*i], + None => catchall, + }; + // Mark the builder as used so that we know to add it to the OCI image + wrapped_builder.used = true; + let clamp_mtime = wrapped_builder.clamp_mtime; + let builder = &mut wrapped_builder.inner; + + if entry.file_type().is_symlink() { + if meta.mtime() > clamp_mtime { + // Setting the mtime on a symlink is fiddly with tar-rs, so we use filetime to change + // the mtime before adding the symlink to the tar archive + let mtime = filetime::FileTime::from_unix_time(clamp_mtime, 0); + filetime::set_symlink_file_times(entry.path(), mtime, mtime)?; + } + add_pax_extension_header(entry.path(), builder)?; + builder.append_path_with_name(entry.path(), rel_path)?; + } else if entry.file_type().is_file() || entry.file_type().is_dir() { + add_pax_extension_header(entry.path(), builder)?; + + // If this is a hardlink, add a link header instead of the file + // if this isn't the first time we've seen this inode + if meta.nlink() > 1 { + match hardlinks.entry((meta.dev(), meta.ino())) { + Entry::Occupied(e) => { + // Add link header and continue to next entry + let mut header = tar::Header::new_gnu(); + header.set_metadata(&meta); + if meta.mtime() > clamp_mtime { + header.set_mtime(clamp_mtime as u64); + } + header.set_entry_type(tar::EntryType::Link); + header.set_cksum(); + builder.append_link(&mut header, &rel_path, e.get())?; + continue; + } + Entry::Vacant(e) => { + // This is the first time we've seen this inode + e.insert(rel_path.clone()); + } + } + } + + let mut header = tar::Header::new_gnu(); + header.set_size(meta.len()); + header.set_metadata(&meta); + if meta.mtime() > clamp_mtime { + header.set_mtime(clamp_mtime as u64); + } + if entry.file_type().is_file() { + builder.append_data( + &mut header, + rel_path, + &mut std::fs::File::open(entry.path())?, + )?; + } else { + builder.append_data(&mut header, rel_path, &mut std::io::empty())?; + }; + } + } + + Ok(()) + } + + /// Finalize the image by writing the layers to the OCI image directory + /// and updating the given manifest and image configuration + fn finish<'a>( + &self, + package_layers: Vec>, + catchall: LayerBuilder<'a>, + ) -> Result { + write::ok("Writing", "image layers")?; + + let mut manifest = self.manifest.clone(); + let mut config = self.config.clone(); + + package_layers + .into_iter() + .filter(|b| b.used) + .try_for_each(|builder| { + let (layer, created_by) = builder.finish()?; + self.oci_dir.push_layer_full( + &mut manifest, + &mut config, + layer, + Option::>::None, + &created_by, + self.creation_time, + ); + Result::<_, anyhow::Error>::Ok(()) + })?; + + if catchall.used { + let (layer, created_by) = catchall.finish()?; + self.oci_dir.push_layer_full( + &mut manifest, + &mut config, + layer, + Option::>::None, + &created_by, + self.creation_time, + ); + } + + write::ok("Writing", "image manifest and config")?; + Ok(self.oci_dir.insert_manifest_and_config( + manifest, + config, + Some(&self.tag), + ocidir::oci_spec::image::Platform::default(), + )?) + } + + fn create_layer( + &self, + created_by: impl Into, + clamp_mtime: i64, + ) -> Result { + let mut inner = tar::Builder::new(LayerWriter::new( + &self.oci_dir, + self.compression_algorithm, + self.compression_level, + )?); + inner.follow_symlinks(false); + Ok(LayerBuilder { + inner, + created_by: created_by.into(), + used: false, + clamp_mtime, + }) + } + + fn most_popular_packages(&self) -> Result> { + Python::with_gil(|py| { + // Resolve is a compiled in python module for resolving dependencies + let _nix_closure_graph = PyModule::from_code_bound( + py, + include_str!("nix_closure_graph.py"), + "nix_closure_graph", + "nix_closure_graph", + )?; + let graph = PyModule::from_code_bound(py, include_str!("graph.py"), "graph", "graph")?; + let args = PyTuple::new_bound( + py, + &[ + self.filesystem_root.to_object(py), + self.max_layers.to_object(py), + self.rpm_size_threshold.to_object(py), + ], + ); + Ok::<_, anyhow::Error>( + graph + .getattr("most_popular_packages")? + .call1(args)? + .extract()?, + ) + }) + .context("Failed to determine layer graph") + } +} + +/// A struct for extracting package information from a hawkey.Package +#[derive(Debug, FromPyObject)] +struct PyPackage { + name: String, + evr: String, + arch: String, + files: Vec, + buildtime: i64, +} + +struct LayerBuilder<'a> { + inner: tar::Builder>, + created_by: String, + used: bool, + /// Directories and symlinks in an RPM may have an mtime of the install time. + /// Whilst rpm respects SOURCE_DATE_EPOCH, we want package layers in independent builds (with different SOURCE_DATE_EPOCHs) + /// to be identical. + clamp_mtime: i64, +} + +impl<'a> LayerBuilder<'a> { + fn finish(self) -> Result<(Layer, String)> { + let layer = self.inner.into_inner()?.complete()?; + Ok((layer, self.created_by)) + } +} + +fn path_to_layer_map(py_pkgs: Vec) -> HashMap { + // Map paths to the index of the layer they belong to + let mut path_to_layer_idx = HashMap::new(); + for (i, pkg) in py_pkgs.into_iter().enumerate() { + for file in pkg.files { + path_to_layer_idx.insert( + file.strip_prefix("/") + .map(|p| p.to_path_buf()) + .unwrap_or(file), + i, + ); + } + } + path_to_layer_idx +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_builder() { + let out = test_temp_dir::test_temp_dir!(); + let cfg = Imager::with_paths("foo", out.as_path_untracked()) + .unwrap() + .build(); + assert_eq!(cfg.max_layers, 125); + } +} diff --git a/src/imager/nix_closure_graph.py b/src/imager/nix_closure_graph.py new file mode 100644 index 0000000..314cdae --- /dev/null +++ b/src/imager/nix_closure_graph.py @@ -0,0 +1,566 @@ +# Taken from https://github.com/NixOS/nixpkgs/blob/69df3ac140f95662ad519f3e453f579409f6e42b/pkgs/build-support/references-by-popularity/closure-graph.py#L408 +# and removed the `main()` invocation. + +# Copyright (c) 2003-2024 Eelco Dolstra and the Nixpkgs/NixOS contributors + +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: + +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +# IMPORTANT: Making changes? +# +# Validate your changes with python3 ./closure-graph.py --test + + +# Using a simple algorithm, convert the references to a path in to a +# sorted list of dependent paths based on how often they're referenced +# and how deep in the tree they live. Equally-"popular" paths are then +# sorted by name. +# +# The existing writeClosure prints the paths in a simple ascii-based +# sorting of the paths. +# +# Sorting the paths by graph improves the chances that the difference +# between two builds appear near the end of the list, instead of near +# the beginning. This makes a difference for Nix builds which export a +# closure for another program to consume, if that program implements its +# own level of binary diffing. +# +# For an example, Docker Images. If each store path is a separate layer +# then Docker Images can be very efficiently transfered between systems, +# and we get very good cache reuse between images built with the same +# version of Nixpkgs. However, since Docker only reliably supports a +# small number of layers (42) it is important to pick the individual +# layers carefully. By storing very popular store paths in the first 40 +# layers, we improve the chances that the next Docker image will share +# many of those layers.* +# +# Given the dependency tree: +# +# A - B - C - D -\ +# \ \ \ \ +# \ \ \ \ +# \ \ - E ---- F +# \- G +# +# Nodes which have multiple references are duplicated: +# +# A - B - C - D - F +# \ \ \ +# \ \ \- E - F +# \ \ +# \ \- E - F +# \ +# \- G +# +# Each leaf node is now replaced by a counter defaulted to 1: +# +# A - B - C - D - (F:1) +# \ \ \ +# \ \ \- E - (F:1) +# \ \ +# \ \- E - (F:1) +# \ +# \- (G:1) +# +# Then each leaf counter is merged with its parent node, replacing the +# parent node with a counter of 1, and each existing counter being +# incremented by 1. That is to say `- D - (F:1)` becomes `- (D:1, F:2)`: +# +# A - B - C - (D:1, F:2) +# \ \ \ +# \ \ \- (E:1, F:2) +# \ \ +# \ \- (E:1, F:2) +# \ +# \- (G:1) +# +# Then each leaf counter is merged with its parent node again, merging +# any counters, then incrementing each: +# +# A - B - (C:1, D:2, E:2, F:5) +# \ \ +# \ \- (E:1, F:2) +# \ +# \- (G:1) +# +# And again: +# +# A - (B:1, C:2, D:3, E:4, F:8) +# \ +# \- (G:1) +# +# And again: +# +# (A:1, B:2, C:3, D:4, E:5, F:9, G:2) +# +# and then paths have the following "popularity": +# +# A 1 +# B 2 +# C 3 +# D 4 +# E 5 +# F 9 +# G 2 +# +# and the popularity contest would result in the paths being printed as: +# +# F +# E +# D +# C +# B +# G +# A +# +# * Note: People who have used a Dockerfile before assume Docker's +# Layers are inherently ordered. However, this is not true -- Docker +# layers are content-addressable and are not explicitly layered until +# they are composed in to an Image. + +import sys +import json +import unittest + +from pprint import pprint +from collections import defaultdict + + +def debug(msg, *args, **kwargs): + if False: + print("DEBUG: {}".format(msg.format(*args, **kwargs)), file=sys.stderr) + + +# Find paths in the original dataset which are never referenced by +# any other paths +def find_roots(closures): + roots = [] + for closure in closures: + path = closure["path"] + if not any_refer_to(path, closures): + roots.append(path) + + return roots + + +class TestFindRoots(unittest.TestCase): + def test_find_roots(self): + self.assertCountEqual( + find_roots( + [ + { + "path": "/nix/store/foo", + "references": ["/nix/store/foo", "/nix/store/bar"], + }, + { + "path": "/nix/store/bar", + "references": ["/nix/store/bar", "/nix/store/tux"], + }, + {"path": "/nix/store/hello", "references": []}, + ] + ), + ["/nix/store/foo", "/nix/store/hello"], + ) + + +def any_refer_to(path, closures): + for closure in closures: + if path != closure["path"]: + if path in closure["references"]: + return True + return False + + +class TestAnyReferTo(unittest.TestCase): + def test_has_references(self): + self.assertTrue( + any_refer_to( + "/nix/store/bar", + [ + {"path": "/nix/store/foo", "references": ["/nix/store/bar"]}, + ], + ), + ) + + def test_no_references(self): + self.assertFalse( + any_refer_to( + "/nix/store/foo", + [ + { + "path": "/nix/store/foo", + "references": ["/nix/store/foo", "/nix/store/bar"], + }, + ], + ), + ) + + +def all_paths(closures): + paths = [] + for closure in closures: + paths.append(closure["path"]) + paths.extend(closure["references"]) + paths.sort() + return list(set(paths)) + + +class TestAllPaths(unittest.TestCase): + def test_returns_all_paths(self): + self.assertCountEqual( + all_paths( + [ + { + "path": "/nix/store/foo", + "references": ["/nix/store/foo", "/nix/store/bar"], + }, + { + "path": "/nix/store/bar", + "references": ["/nix/store/bar", "/nix/store/tux"], + }, + {"path": "/nix/store/hello", "references": []}, + ] + ), + [ + "/nix/store/foo", + "/nix/store/bar", + "/nix/store/hello", + "/nix/store/tux", + ], + ) + + def test_no_references(self): + self.assertFalse( + any_refer_to( + "/nix/store/foo", + [ + { + "path": "/nix/store/foo", + "references": ["/nix/store/foo", "/nix/store/bar"], + }, + ], + ), + ) + + +# Convert: +# +# [ +# { path: /nix/store/foo, references: [ /nix/store/foo, /nix/store/bar, /nix/store/baz ] }, +# { path: /nix/store/bar, references: [ /nix/store/bar, /nix/store/baz ] }, +# { path: /nix/store/baz, references: [ /nix/store/baz, /nix/store/tux ] }, +# { path: /nix/store/tux, references: [ /nix/store/tux ] } +# ] +# +# To: +# { +# /nix/store/foo: [ /nix/store/bar, /nix/store/baz ], +# /nix/store/bar: [ /nix/store/baz ], +# /nix/store/baz: [ /nix/store/tux ] }, +# /nix/store/tux: [ ] +# } +# +# Note that it drops self-references to avoid loops. +def make_lookup(closures): + lookup = {} + + for closure in closures: + # paths often self-refer + nonreferential_paths = [ + ref for ref in closure["references"] if ref != closure["path"] + ] + lookup[closure["path"]] = nonreferential_paths + + return lookup + + +class TestMakeLookup(unittest.TestCase): + def test_returns_lookp(self): + self.assertDictEqual( + make_lookup( + [ + { + "path": "/nix/store/foo", + "references": ["/nix/store/foo", "/nix/store/bar"], + }, + { + "path": "/nix/store/bar", + "references": ["/nix/store/bar", "/nix/store/tux"], + }, + {"path": "/nix/store/hello", "references": []}, + ] + ), + { + "/nix/store/foo": ["/nix/store/bar"], + "/nix/store/bar": ["/nix/store/tux"], + "/nix/store/hello": [], + }, + ) + + +# Convert: +# +# /nix/store/foo with +# { +# /nix/store/foo: [ /nix/store/bar, /nix/store/baz ], +# /nix/store/bar: [ /nix/store/baz ], +# /nix/store/baz: [ /nix/store/tux ] }, +# /nix/store/tux: [ ] +# } +# +# To: +# +# { +# /nix/store/bar: { +# /nix/store/baz: { +# /nix/store/tux: {} +# } +# }, +# /nix/store/baz: { +# /nix/store/tux: {} +# } +# } +subgraphs_cache = {} + + +def make_graph_segment_from_root(root, lookup): + global subgraphs_cache + children = {} + for ref in lookup[root]: + # make_graph_segment_from_root is a pure function, and will + # always return the same result based on a given input. Thus, + # cache computation. + # + # Python's assignment will use a pointer, preventing memory + # bloat for large graphs. + if ref not in subgraphs_cache: + debug("Subgraph Cache miss on {}".format(ref)) + subgraphs_cache[ref] = make_graph_segment_from_root(ref, lookup) + else: + debug("Subgraph Cache hit on {}".format(ref)) + children[ref] = subgraphs_cache[ref] + return children + + +class TestMakeGraphSegmentFromRoot(unittest.TestCase): + def test_returns_graph(self): + self.assertDictEqual( + make_graph_segment_from_root( + "/nix/store/foo", + { + "/nix/store/foo": ["/nix/store/bar"], + "/nix/store/bar": ["/nix/store/tux"], + "/nix/store/tux": [], + "/nix/store/hello": [], + }, + ), + {"/nix/store/bar": {"/nix/store/tux": {}}}, + ) + + def test_returns_graph_tiny(self): + self.assertDictEqual( + make_graph_segment_from_root( + "/nix/store/tux", + { + "/nix/store/foo": ["/nix/store/bar"], + "/nix/store/bar": ["/nix/store/tux"], + "/nix/store/tux": [], + }, + ), + {}, + ) + + +# Convert a graph segment in to a popularity-counted dictionary: +# +# From: +# { +# /nix/store/foo: { +# /nix/store/bar: { +# /nix/store/baz: { +# /nix/store/tux: {} +# } +# } +# /nix/store/baz: { +# /nix/store/tux: {} +# } +# } +# } +# +# to: +# [ +# /nix/store/foo: 1 +# /nix/store/bar: 2 +# /nix/store/baz: 4 +# /nix/store/tux: 6 +# ] +popularity_cache = {} + + +def graph_popularity_contest(full_graph): + global popularity_cache + popularity = defaultdict(int) + for path, subgraph in full_graph.items(): + popularity[path] += 1 + # graph_popularity_contest is a pure function, and will + # always return the same result based on a given input. Thus, + # cache computation. + # + # Python's assignment will use a pointer, preventing memory + # bloat for large graphs. + if path not in popularity_cache: + debug("Popularity Cache miss on {}", path) + popularity_cache[path] = graph_popularity_contest(subgraph) + else: + debug("Popularity Cache hit on {}", path) + + subcontest = popularity_cache[path] + for subpath, subpopularity in subcontest.items(): + debug("Calculating popularity for {}", subpath) + popularity[subpath] += subpopularity + 1 + + return popularity + + +class TestGraphPopularityContest(unittest.TestCase): + def test_counts_popularity(self): + self.assertDictEqual( + graph_popularity_contest( + { + "/nix/store/foo": { + "/nix/store/bar": {"/nix/store/baz": {"/nix/store/tux": {}}}, + "/nix/store/baz": {"/nix/store/tux": {}}, + } + } + ), + { + "/nix/store/foo": 1, + "/nix/store/bar": 2, + "/nix/store/baz": 4, + "/nix/store/tux": 6, + }, + ) + + +# Emit a list of packages by popularity, most first: +# +# From: +# [ +# /nix/store/foo: 1 +# /nix/store/bar: 1 +# /nix/store/baz: 2 +# /nix/store/tux: 2 +# ] +# +# To: +# [ /nix/store/baz /nix/store/tux /nix/store/bar /nix/store/foo ] +def order_by_popularity(paths): + paths_by_popularity = defaultdict(list) + popularities = [] + for path, popularity in paths.items(): + popularities.append(popularity) + paths_by_popularity[popularity].append(path) + + popularities = list(set(popularities)) + popularities.sort() + + flat_ordered = [] + for popularity in popularities: + paths = paths_by_popularity[popularity] + paths.sort(key=package_name) + + flat_ordered.extend(reversed(paths)) + return list(reversed(flat_ordered)) + + +class TestOrderByPopularity(unittest.TestCase): + def test_returns_in_order(self): + self.assertEqual( + order_by_popularity( + { + "/nix/store/foo": 1, + "/nix/store/bar": 1, + "/nix/store/baz": 2, + "/nix/store/tux": 2, + } + ), + ["/nix/store/baz", "/nix/store/tux", "/nix/store/bar", "/nix/store/foo"], + ) + + +def package_name(path): + parts = path.split("-") + start = parts.pop(0) + # don't throw away any data, so the order is always the same. + # even in cases where only the hash at the start has changed. + parts.append(start) + return "-".join(parts) + + +def main(): + filename = sys.argv[1] + key = sys.argv[2] + + debug("Loading from {}", filename) + with open(filename) as f: + data = json.load(f) + + # Data comes in as: + # [ + # { path: /nix/store/foo, references: [ /nix/store/foo, /nix/store/bar, /nix/store/baz ] }, + # { path: /nix/store/bar, references: [ /nix/store/bar, /nix/store/baz ] }, + # { path: /nix/store/baz, references: [ /nix/store/baz, /nix/store/tux ] }, + # { path: /nix/store/tux, references: [ /nix/store/tux ] } + # ] + # + # and we want to get out a list of paths ordered by how universally, + # important they are, ie: tux is referenced by every path, transitively + # so it should be #1 + # + # [ + # /nix/store/tux, + # /nix/store/baz, + # /nix/store/bar, + # /nix/store/foo, + # ] + graph = data[key] + + debug("Finding roots from {}", key) + roots = find_roots(graph) + debug("Making lookup for {}", key) + lookup = make_lookup(graph) + + full_graph = {} + for root in roots: + debug("Making full graph for {}", root) + full_graph[root] = make_graph_segment_from_root(root, lookup) + + debug("Running contest") + contest = graph_popularity_contest(full_graph) + debug("Ordering by popularity") + ordered = order_by_popularity(contest) + debug("Checking for missing paths") + missing = [] + for path in all_paths(graph): + if path not in ordered: + missing.append(path) + + ordered.extend(missing) + print("\n".join(ordered)) diff --git a/src/imager/sha256_writer.rs b/src/imager/sha256_writer.rs new file mode 100644 index 0000000..8ec2842 --- /dev/null +++ b/src/imager/sha256_writer.rs @@ -0,0 +1,56 @@ +//! Copyright (C) Microsoft Corporation. +//! +//! This program is free software: you can redistribute it and/or modify +//! it under the terms of the GNU General Public License as published by +//! the Free Software Foundation, either version 3 of the License, or +//! (at your option) any later version. +//! +//! This program is distributed in the hope that it will be useful, +//! but WITHOUT ANY WARRANTY; without even the implied warranty of +//! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//! GNU General Public License for more details. +//! +//! You should have received a copy of the GNU General Public License +//! along with this program. If not, see . +use openssl::sha::Sha256; +use std::fmt::Write as _; +use std::io::{Result, Write}; + +/// Wraps a writer and calculates the sha256 digest of data written to the inner writer +pub(crate) struct Sha256Writer { + writer: W, + sha: Sha256, +} + +impl Sha256Writer { + pub(crate) fn new(writer: W) -> Self { + Self { + writer, + sha: Sha256::new(), + } + } + + /// Return the hex encoded sha256 digest of the written data, and the underlying writer + pub(crate) fn finish(self) -> (String, W) { + let mut digest = String::new(); + for byte in self.sha.finish().iter() { + write!(digest, "{:02x}", byte).unwrap(); + } + (digest, self.writer) + } +} + +impl Write for Sha256Writer +where + W: Write, +{ + fn write(&mut self, buf: &[u8]) -> Result { + let len = self.writer.write(buf)?; + self.sha.update(&buf[..len]); + Ok(len) + } + + fn flush(&mut self) -> Result<()> { + self.writer.flush() + } +} diff --git a/src/lib.rs b/src/lib.rs index 4480f49..e804535 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,9 +22,9 @@ use std::{ }; use anyhow::{bail, Context}; -mod archive; pub mod cli; pub mod config; +pub mod imager; pub mod lockfile; pub mod subid; pub mod write; diff --git a/src/lockfile/build.rs b/src/lockfile/build.rs index 5d2d21c..9845d83 100644 --- a/src/lockfile/build.rs +++ b/src/lockfile/build.rs @@ -17,23 +17,16 @@ use std::ffi::OsStr; use std::path::Path; use std::{fs, process::Command}; +use super::Lockfile; +use crate::config::Config; +use crate::imager; +use crate::write; use anyhow::{bail, Context, Result}; use chrono::DateTime; -use flate2::Compression; use glob::glob; -use ocidir::oci_spec::image::MediaType; -use ocidir::{new_empty_manifest, OciDir}; use rusqlite::Connection; use tempfile::TempDir; -use super::Lockfile; -use crate::archive::append_dir_all_with_xattrs; -use crate::config::Config; -use crate::write; -use ocidir::cap_std::fs::Dir; - -const CREATED_BY: &str = "Created by rpmoci"; - impl Lockfile { /// Build a container image from a lockfile pub fn build( @@ -44,15 +37,19 @@ impl Lockfile { vendor_dir: Option<&Path>, labels: HashMap, ) -> Result<()> { - // Ensure OCI directory exists - fs::create_dir_all(image) - .context(format!("Failed to create OCI image directory `{}`", &image))?; - let dir = Dir::open_ambient_dir(image, ocidir::cap_std::ambient_authority()) - .context("Failed to open image directory")?; - let oci_dir = OciDir::ensure(&dir)?; - let creation_time = creation_time()?; - let installroot = TempDir::new()?; // This needs to outlive the layer builder below. + let installroot = TempDir::new()?; // This needs to outlive the image builder below. + let image_config = cfg + .image + .to_oci_image_configuration(labels, creation_time)?; + + // Create the image writer early to ensure the image directory is created successfully + let image_builder = imager::Imager::with_paths(installroot.path(), image)? + .creation_time(creation_time) + .config(image_config) + .tag(tag) + .build(); + if let Some(vendor_dir) = vendor_dir { // Use vendored RPMs rather than downloading self.create_installroot(installroot.path(), vendor_dir, false, cfg, &creation_time) @@ -69,39 +66,8 @@ impl Lockfile { } .context("Failed to create installroot")?; - // Create the root filesystem layer - write::ok("Creating", "root filesystem layer")?; - let mut builder = oci_dir.create_layer(Compression::fast().into())?; - builder.follow_symlinks(false); - append_dir_all_with_xattrs(&mut builder, installroot.path(), creation_time.timestamp()) - .context("failed to archive root filesystem")?; - let layer = builder.into_inner()?.complete()?; - - // Create the image configuration blob - write::ok("Writing", "image configuration blob")?; - let mut image_config = cfg - .image - .to_oci_image_configuration(labels, creation_time)?; - // Create the image manifest - let mut manifest = new_empty_manifest() - .media_type(MediaType::ImageManifest) - .build()?; - oci_dir.push_layer_full( - &mut manifest, - &mut image_config, - layer, - Option::>::None, - CREATED_BY, - creation_time, - ); + image_builder.create_image()?; - write::ok("Writing", "image manifest and config")?; - oci_dir.insert_manifest_and_config( - manifest, - image_config, - Some(tag), - ocidir::oci_spec::image::Platform::default(), - )?; Ok(()) }