diff --git a/.gitignore b/.gitignore index 38c039cc..9dacce27 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ result* .secret cosign.key + +# Used for testing purposes +!next/tests/repo/cosign.key !test-files/keys/cosign.key # Local testing for bluebuild recipe files diff --git a/Cargo.lock b/Cargo.lock index 14e91f86..a3e7ae88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -472,6 +472,7 @@ dependencies = [ name = "blue-build" version = "0.9.37" dependencies = [ + "blue-build-next-builder", "blue-build-process-management", "blue-build-recipe", "blue-build-template", @@ -512,6 +513,30 @@ dependencies = [ "yaml_serde", ] +[[package]] +name = "blue-build-next-builder" +version = "0.9.37" +dependencies = [ + "blue-build-process-management", + "blue-build-recipe", + "blue-build-utils", + "bon", + "cached 2.0.2", + "comlexr", + "futures", + "lazy-regex", + "log", + "miette", + "oci-client 0.17.0", + "rayon", + "rstest", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.20", + "tokio", +] + [[package]] name = "blue-build-process-management" version = "0.9.37" @@ -524,6 +549,7 @@ dependencies = [ "clap", "colored", "comlexr", + "futures", "indicatif", "indicatif-log-bridge", "lazy-regex", @@ -556,12 +582,14 @@ dependencies = [ "bon", "cached 2.0.2", "colored", + "comlexr", "indexmap 2.14.0", "log", "miette", "oci-client 0.17.0", "rstest", "serde", + "serde_json", "structstruck", "yaml_serde", ] diff --git a/Cargo.toml b/Cargo.toml index 0d278ce7..800229eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,11 @@ [workspace] -members = ["utils", "recipe", "template", "process"] +members = [ + "utils", + "recipe", + "template", + "process", + "next" +] [workspace.package] description = "A CLI tool built for creating Containerfile templates for ostree based atomic distros" @@ -16,6 +22,7 @@ chrono = "=0.4.45" clap = "=4.6.6" colored = "=3.1.1" comlexr = "=1.5.0" +futures = "0.3.34" indexmap = { version = "=2.14.0", features = ["serde"] } indicatif = { version = "=0.18.6", features = ["improved_unicode", "rayon"] } lazy-regex = "=3.6.1" @@ -37,6 +44,7 @@ shadow-rs = { version = "=2.0.0", default-features = false } structstruck = "=0.5.1" syntect = { version = "=5.3.0", default-features = false, features = ["default-fancy"] } tempfile = "=3.27.0" +thiserror = "=2.0.20" tokio = { version = "=1.53.1", default-features = false } uuid = { version = "=1.25.0", features = ["v4"] } which = "=8.0.5" @@ -68,8 +76,8 @@ license.workspace = true [package.metadata.release] pre-release-hook = ["git", "cliff", "-o", "CHANGELOG.md", "--tag", "{{version}}"] pre-release-replacements = [ - { file = "install.sh", search = "VERSION=v\\d+\\.\\d+\\.\\d+", replace = "VERSION=v{{version}}" }, - { file = "flake.nix", search = "version = \"v\\d+\\.\\d+\\.\\d+\";", replace = "version = \"v{{version}}\";" } + { file = "install.sh", search = "VERSION=v\\d+\\.\\d+\\.\\d+", replace = "VERSION=v{{version}}" }, + { file = "flake.nix", search = "version = \"v\\d+\\.\\d+\\.\\d+\";", replace = "version = \"v{{version}}\";" } ] [dependencies] @@ -77,6 +85,7 @@ blue-build-recipe = { version = "=0.9.37", path = "./recipe" } blue-build-template = { version = "=0.9.37", path = "./template" } blue-build-utils = { version = "=0.9.37", path = "./utils" } blue-build-process-management = { version = "=0.9.37", path = "./process" } +blue-build-next-builder = { version = "=0.9.37", path = "./next" } clap-verbosity-flag = "=3.0.4" clap_complete = "=4.6.9" clap_complete_nushell = "=4.6.2" @@ -86,7 +95,6 @@ open = "=5.4.1" os_info = "=3.15.0" requestty = { version = "=0.6.3", features = ["macros", "termion"] } rust-embed = { version = "=8.12.0", features = ["debug-embed", "compression", "deterministic-timestamps"] } -thiserror = "=2.0.20" urlencoding = "=2.1.3" yaml-rust2 = "=0.12.0" @@ -106,6 +114,7 @@ serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true shadow-rs.workspace = true +thiserror.workspace = true tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } bon.workspace = true diff --git a/Earthfile b/Earthfile index 336c80a1..fdbbfdd1 100644 --- a/Earthfile +++ b/Earthfile @@ -135,6 +135,7 @@ common: utils/ \ process/ \ scripts/ \ + next/ \ .git/ \ /app diff --git a/bacon.toml b/bacon.toml index 3c5ead85..a2b62fbc 100644 --- a/bacon.toml +++ b/bacon.toml @@ -11,13 +11,13 @@ default_job = "clippy-all" command = ["cargo", "check", "--tests", "--color", "always"] need_stdout = false default_watch = false -watch = ["src", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] +watch = ["src", "next", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] [jobs.check-all] command = ["cargo", "check", "--all-features", "--tests", "--color", "always"] need_stdout = false default_watch = false -watch = ["src", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] +watch = ["src", "next", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] [jobs.clippy] command = [ @@ -26,17 +26,23 @@ command = [ ] need_stdout = false default_watch = false -watch = ["src", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] +watch = ["src", "next", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] [jobs.clippy-all] command = [ - "cargo", "clippy", "--workspace", + "cargo", + # "+nightly", + "clippy", + "--workspace", "--all-features", - "--tests", "--color", "always", + # "--verbose", + "--tests", + "--color", + "always", ] need_stdout = false default_watch = false -watch = ["src", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] +watch = ["src", "next", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] [jobs.test] command = [ @@ -45,7 +51,7 @@ command = [ ] need_stdout = true default_watch = false -watch = ["src", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs", "test-files", "integration-tests"] +watch = ["src", "next", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs", "test-files", "integration-tests"] [jobs.test-all] command = [ @@ -54,13 +60,13 @@ command = [ ] need_stdout = true default_watch = false -watch = ["src", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs", "test-files", "integration-tests"] +watch = ["src", "next", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs", "test-files", "integration-tests"] [jobs.doc] command = ["cargo", "doc", "--color", "always"] need_stdout = false default_watch = false -watch = ["src", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] +watch = ["src", "next", "process", "recipe", "template", "utils", "Cargo.toml", "build.rs"] # If the doc compiles, then it opens in your browser and bacon switches # to the previous job @@ -74,14 +80,14 @@ command = ["cargo", "install", "--path", ".", "--debug", "--locked", "--color", need_stdout = false allow_warnings = true default_watch = false -watch = ["src", "process", "recipe", "template", "utils", "scripts", "Cargo.toml", "build.rs"] +watch = ["src", "next", "process", "recipe", "template", "utils", "scripts", "Cargo.toml", "build.rs"] [jobs.install-all] command = ["cargo", "install", "--all-features", "--path", ".", "--debug", "--locked", "--color", "always"] need_stdout = false allow_warnings = true default_watch = false -watch = ["src", "process", "recipe", "template", "utils", "scripts", "Cargo.toml", "build.rs"] +watch = ["src", "next", "process", "recipe", "template", "utils", "scripts", "Cargo.toml", "build.rs"] # You may define here keybindings that would be specific to # a project, for example a shortcut to launch a specific job. diff --git a/next/Cargo.toml b/next/Cargo.toml new file mode 100644 index 00000000..f20a91e8 --- /dev/null +++ b/next/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "blue-build-next-builder" +description.workspace = true +edition.workspace = true +repository.workspace = true +categories.workspace = true +version.workspace = true +license = "LGPL-2.1" +build = "build.rs" + +[dependencies] +blue-build-process-management = { version = "=0.9.37", path = "../process" } +blue-build-utils = { version = "=0.9.37", path = "../utils" } +blue-build-recipe = { version = "=0.9.37", path = "../recipe" } + +bon.workspace = true +cached.workspace = true +comlexr.workspace = true +futures.workspace = true +lazy-regex.workspace = true +log.workspace = true +miette.workspace = true +oci-client.workspace = true +rayon.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } + +[dev-dependencies] +rstest.workspace = true +tokio = { workspace = true, features = ["test-util"] } + +[lints] +workspace = true diff --git a/next/LGPL_2.1.txt b/next/LGPL_2.1.txt new file mode 100644 index 00000000..f932d7aa --- /dev/null +++ b/next/LGPL_2.1.txt @@ -0,0 +1,17 @@ +The next generation build engine for BlueBuild. +Copyright (C) 2026 BlueBuild Authors + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 +USA diff --git a/next/LICENSE b/next/LICENSE new file mode 100644 index 00000000..a97c0208 --- /dev/null +++ b/next/LICENSE @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + The next generation build engine for BlueBuild. + Copyright (C) 2026 BlueBuild Authors + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 + USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random + Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/next/build.rs b/next/build.rs new file mode 100644 index 00000000..64975918 --- /dev/null +++ b/next/build.rs @@ -0,0 +1,36 @@ +use std::{ + env, + fs::{self, DirEntry}, + path::PathBuf, +}; + +fn main() { + println!("cargo:rerun-if-changed=scripts/"); + let entries = fs::read_dir("scripts/") + .unwrap() + .collect::, _>>() + .unwrap(); + let scripts = entries + .iter() + .filter(|entry| { + entry.path().is_file() && entry.path().extension().is_some_and(|ext| ext == "sh") + }) + .map(DirEntry::path) + .map(|path| { + format!( + r#"("{path}", include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/scripts/{path}"))),"#, + path = path.file_name().unwrap().display() + ) + }) + .collect::>(); + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("build_scripts.rs"); + fs::write( + out_path, + format!( + "const BUILD_SCRIPTS: [(&str, &[u8]); {}] = [\n{}\n];", + scripts.len(), + scripts.join("\n"), + ), + ) + .unwrap(); +} diff --git a/next/scripts/exports.sh b/next/scripts/exports.sh new file mode 100755 index 00000000..5f35b365 --- /dev/null +++ b/next/scripts/exports.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +# Function to retrieve module configs and populate an array +# Arguments: +# 1. Variable name to store result +# 2. jq query +# 3. Module config content +get_json_array() { + local -n arr="${1}" + local jq_query="${2}" + local module_config="${3}" + + if [[ -z "${jq_query}" || -z "${module_config}" ]]; then + echo "Usage: get_json_array VARIABLE_TO_STORE_RESULTS JQ_QUERY MODULE_CONFIG" >&2 + return 1 + fi + + readarray -t arr < <(echo "${module_config}" | jq -c -r "${jq_query}") +} + +color_string() { + local string="${1}" + local color_code="${2}" + local reset_code="\033[0m" + + # ANSI color codes: https://en.wikipedia.org/wiki/ANSI_escape_code#Colors + # Example color codes: 31=red, 32=green, 33=yellow, 34=blue, 35=magenta, 36=cyan, 37=white + + # Check if color code is provided, otherwise default to white (37) + if [[ -z "${color_code}" ]]; then + color_code="37" + fi + + # Determine if we should force color + if [ -n "${FORCE_COLOR:-}" ] || [ -n "${CLICOLOR_FORCE:-}" ]; then + # Force color: Apply color codes regardless of whether output is a TTY + echo -e "\033[${color_code}m${string}${reset_code}" + elif [ -t 1 ]; then + # Output is a TTY and color is not forced: Apply color codes + echo -e "\033[${color_code}m${string}${reset_code}" + else + # Output is not a TTY: Do not apply color codes + echo "${string}" + fi +} + +feature_enabled() { + # Ensure the function is called with exactly one argument + if [ "$#" -ne 1 ]; then + echo "Usage: feature_enabled " >&2 + return 1 + fi + + local feature="$1" + local -a features + + # Split BB_BUILD_FEATURES by commas and read into an array + IFS=, + read -r -a features <<< "$BB_BUILD_FEATURES" + + # Loop through the array and check for a match + for f in "${features[@]}"; do + # Trim leading and trailing whitespace + local trimmed_f="${f## }" + trimmed_f="${trimmed_f%% }" + + if [[ "$trimmed_f" == "$feature" ]]; then + return 0 + fi + done + + # Feature not found + return 1 +} + +# Parse OS version and export it +export OS_VERSION="$(awk -F= '/^VERSION_ID=/ {gsub(/"/, "", $2); print $2}' /usr/lib/os-release)" +case "$TARGETARCH" in + "amd64") + OS_ARCH="x86_64" + ;; + "arm64") + OS_ARCH="aarch64" + ;; + *) + OS_ARCH="$TARGETARCH" + ;; +esac +export OS_ARCH + +# Export functions for use in sub-shells or sourced scripts +export -f get_json_array + +mkdir -p /var/roothome /var/opt /var/lib/alternatives /var/opt /var/usrlocal diff --git a/next/scripts/post_build.sh b/next/scripts/post_build.sh new file mode 100755 index 00000000..f01d8709 --- /dev/null +++ b/next/scripts/post_build.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -euo pipefail +. /scripts/exports.sh + +shopt -s nullglob + +optfix_dir="/usr/lib/opt" +# needs nullglob, so that this array is empty if /opt is empty +optdirs=("${optfix_dir}"/*) # returns a list of directories in /opt +if [[ -n "${optdirs[*]}" ]]; then + echo "Creating symlinks to fix packages that installed to /opt:" + for optdir in "${optdirs[@]}"; do + opt=$(basename "${optdir}") + lib_opt_dir="${optfix_dir}/${opt}" + link_opt_dir="/opt/${opt}" + echo "Linking ${link_opt_dir} => ${lib_opt_dir}" + echo "L+? \"${link_opt_dir}\" - - - - ${lib_opt_dir}" | tee "/usr/lib/tmpfiles.d/99-bluebuild-optfix-${opt}.conf" + done +fi + +rm -rf /tmp/* /var/* /opt +ln -fs /var/opt /opt + +# Relink rpm-ostree-base-db to rpmdb to ensure it correctly reflects the system +# image's rpmdb and doesn't carry over package info from the base image. +# See: https://github.com/coreos/rpm-ostree/issues/4554 +for file in rpmdb.sqlite rpmdb.sqlite-shm rpmdb.sqlite-wal; do + target="/usr/share/rpm/${file}" + link_path="/usr/lib/sysimage/rpm-ostree-base-db/${file}" + if [[ -f "${target}" && -f "${link_path}" ]]; then + # Note, this needs to be a hardlink, not a symbolic link. + ln -f "${target}" "${link_path}" + fi +done + +# if feature_enabled "bootc" && command -v bootc > /dev/null; then +# bootc container lint +# fi diff --git a/next/scripts/pre_build.sh b/next/scripts/pre_build.sh new file mode 100755 index 00000000..d1ab7a3a --- /dev/null +++ b/next/scripts/pre_build.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +set -eu + +/scripts/setup.sh + +optfix_dir="/usr/lib/opt" + +echo "Preparing system for optfix..." +mkdir -pv "${optfix_dir}" + +if [ -d /opt ] || [ -h /opt ]; then + if ls -A /opt/* 2>/dev/null; then + echo "Moving all /opt/* into ${optfix_dir}" + mv -v /opt/* "${optfix_dir}" + fi + rm -fr /opt +fi + +echo "Linking /opt => ${optfix_dir}" +ln -fs "${optfix_dir}" /opt diff --git a/next/scripts/run_module.sh b/next/scripts/run_module.sh new file mode 100755 index 00000000..cb90a07d --- /dev/null +++ b/next/scripts/run_module.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source /tmp/scripts/exports.sh + +# Function to print a centered text banner within a specified width +print_banner() { + local term_width=80 + + local text=" ${1} " # Text to print + local padding="$(printf '%0.1s' '='{1..600})" + local padlen=0 + + if (( ${#text} < term_width )); then + padlen=$(( (term_width - ${#text}) / 2 )) + fi + + printf '%*.*s%s%*.*s\n' 0 "$padlen" "$padding" "$text" 0 "$padlen" "$padding" +} + +get_script_path() { + local script_name="$1" + local extensions=("nu" "sh" "bash") + local base_script_path="/tmp/modules/${script_name}/${script_name}" + local tried_scripts=() + + # See if + if [[ -f "${base_script_path}" ]]; then + echo "${base_script_path}" + return 0 + fi + tried_scripts+=("${script_name}") + + # Iterate through each extension and check if the file exists + for ext in "${extensions[@]}"; do + local script_path="${base_script_path}.${ext}" + tried_scripts+=("${script_name}.${ext}") + + if [[ -f "$script_path" ]]; then + # Output only the script path without extra information + echo "$script_path" + return 0 # Exit the function when the first matching file is found + fi + done + + # If no matching file was found + echo "Failed to find scripts matching: ${tried_scripts[*]}" >&2 + return 1 +} + +module="$1" +params="$2" +script_path="$(get_script_path "$module")" + +export PATH="/usr/libexec/bluebuild/nu/:$PATH" + +color_string "$(print_banner "Start '${module}' Module")" "33" +chmod +x "${script_path}" + +if "${script_path}" "${params}"; then + color_string "$(print_banner "End '${module}' Module")" "32" + +else + color_string "$(print_banner "Failed '${module}' Module")" "31" + exit 1 +fi diff --git a/next/scripts/setup.sh b/next/scripts/setup.sh new file mode 100755 index 00000000..f9b4078c --- /dev/null +++ b/next/scripts/setup.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# TODO: Remove once we're all POSIX https://github.com/blue-build/modules/issues/503 +command_exists() { + [ -x "/usr/bin/$1" ] || [ -x "/bin/$1" ] || [ -x "/usr/sbin/$1" ] || [ -x "/sbin/$1" ] +} + +if ! command_exists bash \ + || ! command_exists jq \ + || ! command_exists curl \ + || ! command_exists grep \ + || ! command_exists ls; then + if command_exists dnf5; then + dnf5 -y install bash curl coreutils jq grep + elif command_exists dnf4; then + dnf4 -y install bash curl coreutils jq grep + elif command_exists zypper; then + zypper --non-interactive install --auto-agree-with-licenses bash curl coreutils jq grep find + elif command_exists pacman; then + pacman --sync --noconfirm --refresh --sysupgrade bash curl coreutils jq grep + elif command_exists apt-get; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get -y install bash curl coreutils jq grep + elif command_exists apk; then + apk add --no-cache bash curl coreutils grep jq + fi +fi diff --git a/next/src/build_scripts.rs b/next/src/build_scripts.rs new file mode 100644 index 00000000..d6771a2a --- /dev/null +++ b/next/src/build_scripts.rs @@ -0,0 +1,50 @@ +use std::{fs, iter::once}; + +use blue_build_utils::tempdir; +use cached::once; +use comlexr::cmd; +use miette::{IntoDiagnostic, Result}; + +use crate::layer::{FinalizeLayer, FromLayer, Layer, LayerCommand, LayerId, UnshareLayer}; + +include!(concat!(env!("OUT_DIR"), "/build_scripts.rs")); + +/// This produces the layer containing the embeded +/// build scripts that are used as a harness for +/// executing the modules. +#[once] +pub async fn build_scripts_layer() -> Result { + let temp_dir = tempdir()?; + let files = BUILD_SCRIPTS + .iter() + .map(|(file, contents)| { + let path = temp_dir.path().join(file); + fs::write(&path, contents).into_diagnostic()?; + Ok((path, file)) + }) + .collect::>>()?; + let from = FromLayer::builder().build(); + Layer::from( + UnshareLayer::builder() + .parent(from) + .commands( + once(LayerCommand::from(cmd!( + "mkdir", + "-p", + "${BB_UNSHARE_MOUNT}/scripts" + ))) + .chain( + files + .iter() + .flat_map(|(src, dest)| { + let dest = format!("${{BB_UNSHARE_MOUNT}}/scripts/{dest}"); + [cmd!("cp", "-f", src, &dest), cmd!("chmod", "+x", dest)] + }) + .map(LayerCommand::from), + ), + ) + .build(), + ) + .finalize() + .await +} diff --git a/next/src/layer.rs b/next/src/layer.rs new file mode 100644 index 00000000..bc357add --- /dev/null +++ b/next/src/layer.rs @@ -0,0 +1,277 @@ +mod build_step; +mod copy; +mod from; +mod mount; +mod run; +mod unshare; + +use std::{ + collections::BTreeMap, + ffi::{OsStr, OsString}, + hash::Hash, + process::Command, + str::FromStr, + sync::Arc, +}; + +use blue_build_utils::cmd_out; +use comlexr::{cmd, cmd_mut}; +use lazy_regex::regex; +use miette::{Diagnostic, IntoDiagnostic, Result}; +use serde::Deserialize; +use thiserror::Error; + +use crate::layer::build_step::{Container, ReadyBuildStep, StepBuilder}; + +pub use copy::*; +pub use from::*; +pub use mount::*; +pub use run::*; +pub use unshare::*; + +/// Build a `Layer`. +trait BuildLayer: Sized { + /// This function runs the `buildah` command + /// for the `Layer` type. + async fn run(&self, container: &Container) -> Result<()>; + + /// Build the image, producing a `BuildStep`. + async fn build(&self) -> Result; +} + +/// Finalize a build and return the `LayerId` +/// of the image. +#[expect(private_bounds)] +pub trait FinalizeLayer: BuildLayer { + /// Finalize the layer by running the build graph + /// and returning the `LayerId` that can be used + /// to create another build graph. + /// + /// # Errors + /// Will error if the build fails. + #[expect(async_fn_in_trait)] + async fn finalize(self) -> Result { + let step = self.build().await?; + step.finalize().await + } +} + +impl FinalizeLayer for Arc {} +impl FinalizeLayer for Arc {} +impl FinalizeLayer for Arc {} + +/// An enumeration of all the possible `Layer`s +/// that can be used in the build graph. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Layer { + /// The `FromLayer`. + From(Arc), + + /// The `RunLayer`. + Run(Arc), + + /// The `CopyLayer`. + Copy(Arc), + + /// The `UnshareLayer` + Unshare(Arc), +} + +impl FinalizeLayer for Layer {} + +impl Layer { + fn get_root(&self) -> Arc { + match self { + Self::From(from) => from.clone(), + Self::Run(run) => run.parent().get_root(), + Self::Copy(copy) => copy.parent().get_root(), + Self::Unshare(unshare) => unshare.parent().get_root(), + } + } +} + +macro_rules! layers { + ($($typ:ty => $var:ident),* $(,)?) => { + impl BuildLayer for Layer { + async fn build(&self) -> Result { + match self { + $(Self::$var(layer) => Box::pin(layer.build()).await,)* + } + } + + async fn run(&self, container: &Container) -> Result<()> { + match self { + $(Self::$var(layer) => layer.run(container).await,)* + } + } + } + + $( + impl From> for Layer { + fn from(value: Arc<$typ>) -> Self { + Self::$var(value) + } + } + + impl From<$typ> for Layer { + fn from(value: $typ) -> Self { + Self::$var(Arc::new(value)) + } + } + )* + }; +} + +layers!( + FromLayer => From, + RunLayer => Run, + CopyLayer => Copy, + UnshareLayer => Unshare, +); + +/// The ID of a commited `Layer`. +/// +/// Can be used for mounts, other images, etc. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct LayerId(Arc); + +#[bon::bon] +impl LayerId { + #[builder] + async fn new(#[builder(into)] container: Container) -> Result { + tokio::task::spawn_blocking(move || { + let layer = cmd_out!( + parse = Self; + err_msg = format!("Failed to commit layer for {container}"); + "buildah", + "commit", + "--rm", + &container, + )?; + drop(container); + Ok(layer) + }) + .await + .into_diagnostic()? + } + + // #[builder(finish_fn = "build")] + // fn from_forked(container: &ForkedContainer) -> Result { + // let layer = cmd + // } +} + +/// The error for a badly parsed `LayerId`. +#[derive(Error, Diagnostic, Debug)] +#[error("Invalid layer ID {}", .0)] +pub struct LayerIdParseError(String); + +impl FromStr for LayerId { + type Err = LayerIdParseError; + + fn from_str(value: &str) -> std::prelude::v1::Result { + let value = value.trim(); + + if regex!("^[a-f0-9]{64}$").is_match(value) { + Ok(Self(Arc::new(value.trim().to_string()))) + } else { + Err(LayerIdParseError(value.to_string())) + } + } +} + +impl std::fmt::Display for LayerId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl<'de> Deserialize<'de> for LayerId { + fn deserialize(deserializer: D) -> std::prelude::v1::Result + where + D: serde::Deserializer<'de>, + { + String::deserialize(deserializer)? + .trim() + .parse() + .map_err(serde::de::Error::custom) + } +} + +impl AsRef for LayerId { + fn as_ref(&self) -> &OsStr { + (*self.0).as_ref() + } +} + +/// A clonable type to store in `Layer` objets. +/// +/// Can be easily converted back and forth +/// from a `Command` object. By using the `Command` +/// type, this allows us to create a `ToString` +/// implementation out of the `Debug` print. +/// `Command`'s `Debug` print is shell-ready and +/// does all the necessary escapes for quotes. +/// +/// If you need to run a command inside a shell, +/// you can build something easily using `comlexr`: +/// +/// ```rust +/// # use blue_build_next_builder::layer::LayerCommand; +/// let shell = LayerCommand::from( +/// comlexr::cmd!( +/// "/bin/sh", +/// "-c", +/// LayerCommand::from( +/// comlexr::cmd!("echo", "I'm running a command") +/// ).to_string(), +/// ) +/// ); +/// ``` +/// +/// The only properties of a `Command` that are retained +/// are `program`, `args`, and `envs`. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct LayerCommand { + program: OsString, + args: Vec, + envs: BTreeMap, +} + +impl From for LayerCommand { + fn from(value: Command) -> Self { + Self { + program: value.get_program().to_owned(), + args: value.get_args().map(ToOwned::to_owned).collect(), + envs: value + .get_envs() + .filter_map(|(key, value)| value.map(|value| (key.to_owned(), value.to_owned()))) + .collect(), + } + } +} + +impl From<&LayerCommand> for Command { + fn from(value: &LayerCommand) -> Self { + let mut c = cmd!( + &value.program, + for &value.args, + ); + + for (key, value) in &value.envs { + cmd_mut!( + env { + key: value, + }; + &mut c, + ); + } + c + } +} + +impl std::fmt::Display for LayerCommand { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", Command::from(self)) + } +} diff --git a/next/src/layer/build_step.rs b/next/src/layer/build_step.rs new file mode 100644 index 00000000..e0589f57 --- /dev/null +++ b/next/src/layer/build_step.rs @@ -0,0 +1,139 @@ +#![expect(refining_impl_trait_internal)] +mod container; +mod layered_step; +mod squashed_step; +mod states { + /// When the build is ready to begin. + pub struct ReadyState; + + /// When the build is running layered. + pub struct RunningLayeredState; + + /// When the build is running squashed. + pub struct RunningSquashedState; + + /// Trait used by running states. + pub trait RunningState {} + impl RunningState for RunningLayeredState {} + impl RunningState for RunningSquashedState {} +} + +use std::marker::PhantomData; + +use miette::Result; + +use crate::layer::{ + Layer, LayerId, + build_step::{squashed_step::SquashedStep, states::RunningState}, +}; + +pub use container::Container; +pub use layered_step::LayeredStep; + +/// Actions that can be taken on a `BuildStep` +/// that is in a `ReadyState`. +pub(super) trait StepBuilder { + /// Runs the build step by + /// + async fn run_build_step(self, layer: Layer) -> Result; + + /// Finalizes the build returning a `LayerId` that + /// can be used for a mount, stage, or another build. + async fn finalize(self) -> Result; +} + +/// Commit the `BuildStep`. +trait StepCommiter { + /// Commits the `Container` and returns a `Buildable` `BuildStep`. + async fn commit(self, container: Container) -> Result; +} + +/// Create a `Container` from a ready `BuildStep`. +trait StepContainer { + /// Create the main `Container` for the `BuildStep` + /// for the layer. Returns an empty `BuildStep` in a `RunningState`. + /// + /// Use the returned `BuildStep` to commit the `Container`. + async fn container(self) -> Result<(Container, BuildStep<(), impl RunningState>)>; +} + +/// A type-based state machine used +/// to track layer and container references +/// while performing build operations. +/// +/// The state machine aspect of this type +/// prevents the build from being misused +/// by making a `Layer` retrieve a `Container` +/// from it (via `CreateContainer`), then requires +/// passing it back in to commit (via `Commiter`). +#[derive(Debug)] +pub struct BuildStep { + base: Base, + _state: PhantomData, +} + +pub enum ReadyBuildStep { + Layer(LayeredStep), + Squash(SquashedStep), +} + +macro_rules! impl_ready { + ($($var:ident => $typ:ty),* $(,)+) => { + $( + impl From<$typ> for ReadyBuildStep { + fn from(value: $typ) -> Self { + Self::$var(value) + } + } + )* + + impl StepBuilder for ReadyBuildStep { + async fn run_build_step(self, layer: Layer) -> Result { + match self { + $(Self::$var(val) => val.run_build_step(layer).await,)* + } + } + + async fn finalize(self) -> Result { + match self { + $(Self::$var(val) => val.finalize().await,)* + } + } + } + }; +} + +impl_ready!( + Layer => LayeredStep, + Squash => SquashedStep, +); + +// pub enum RunningBuildStep { +// Layer(RunningLayeredStep), +// Squash(RunningSquashedStep), +// } + +// macro_rules! impl_running { +// ($($var:ident => $typ:ty),* $(,)+) => { +// $( +// impl From<$typ> for RunningBuildStep { +// fn from(value: $typ) -> Self { +// Self::$var(value) +// } +// } +// )* + +// impl StepCommiter for RunningBuildStep { +// async fn commit(self, container: Container) -> Result { +// match self { +// $(Self::$var(val) => val.commit(container).await,)* +// } +// } +// } +// }; +// } + +// impl_running!( +// Layer => RunningLayeredStep, +// Squash => RunningSquashedStep, +// ); diff --git a/next/src/layer/build_step/container.rs b/next/src/layer/build_step/container.rs new file mode 100644 index 00000000..a7671297 --- /dev/null +++ b/next/src/layer/build_step/container.rs @@ -0,0 +1,110 @@ +use std::{convert::Infallible, ffi::OsStr, str::FromStr}; + +use blue_build_utils::{cmd_out, platform::Platform}; +use miette::{IntoDiagnostic, Result}; +use serde::Deserialize; + +use crate::layer::{FromLayerReference, LayerId}; + +/// A container that is used to run a `Layer` instruction. +#[derive(Debug, PartialEq, Eq, Hash, Deserialize)] +pub struct Container(String); + +#[bon::bon] +impl Container { + /// Create a `Container` from a `FromLayerReference`. + #[builder(finish_fn = "build")] + pub(super) fn from_image( + /// The reference from which to create the container. + #[builder(into)] + image: &FromLayerReference, + + /// The platform for the `Container`. + platform: Option, + ) -> Result { + cmd_out!( + parse = Self; + err_msg = format!( + "Failed to create FROM {image}" + ); + "buildah", + "from", + if let Some(platform) = platform => format!("--platform={platform}"), + image.to_string(), + ) + } + + /// Create a `Container` from a `LayerId`. + pub(super) async fn from_layer(id: &LayerId) -> Result { + tokio::task::spawn_blocking({ + let id = id.clone(); + move || { + cmd_out!( + parse = Self; + err_msg = format!("Unable to create a new container from {id}"); + "buildah", + "from", + &id, + ) + } + }) + .await + .into_diagnostic()? + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } + + // /// Remove the `Container`. + // pub(super) fn remove(self) -> Result<()> { + // cmd_out!( + // err_msg = format!("Failed to remove container {self}"); + // "buildah", + // "rm", + // &self + // ) + // } +} + +impl AsRef for Container { + fn as_ref(&self) -> &OsStr { + self.0.as_ref() + } +} + +impl std::fmt::Display for Container { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl FromStr for Container { + type Err = Infallible; + + fn from_str(s: &str) -> std::prelude::v1::Result { + Ok(Self(s.trim().to_string())) + } +} + +// pub struct ForkedContainer(String); + +// impl ForkedContainer { +// pub fn from_layer(id: &LayerId) -> Result { +// cmd_out!( +// parse = Self; +// err_msg = format!("Unable to create a new container from {id}"); +// "buildah", +// "from", +// id, +// ) +// } +// } + +// impl FromStr for ForkedContainer { +// type Err = Infallible; + +// fn from_str(s: &str) -> std::prelude::v1::Result { +// Ok(Self(s.trim().to_string())) +// } +// } diff --git a/next/src/layer/build_step/layered_step.rs b/next/src/layer/build_step/layered_step.rs new file mode 100644 index 00000000..ea9967fa --- /dev/null +++ b/next/src/layer/build_step/layered_step.rs @@ -0,0 +1,111 @@ +use std::{hash::Hash, marker::PhantomData}; + +use blue_build_utils::platform::Platform; +use miette::Result; + +use crate::layer::{ + BuildLayer, FromLayerReference, Layer, LayerId, StepBuilder, + build_step::{ + BuildStep, Container, ReadyBuildStep, StepCommiter, StepContainer, + states::{ReadyState, RunningLayeredState}, + }, +}; + +/// A `BuildStep` for layer based builds. +pub type LayeredStep = BuildStep; + +/// A `BuildStep` in a `RunningState` for layer based builds. +pub type RunningLayeredStep = BuildStep<(), RunningLayeredState>; + +#[bon::bon] +impl LayeredStep { + #[builder(finish_fn = "build")] + pub async fn new_layered( + image: impl AsRef, + platform: Option, + ) -> Result { + Ok(Self { + base: LayerId::builder() + .container( + Container::from_image() + .image(image.as_ref()) + .maybe_platform(platform) + .build()?, + ) + .build() + .await?, + _state: PhantomData, + }) + } +} + +impl From> for LayeredStep { + fn from(value: Box) -> Self { + Self { + base: value.base, + _state: PhantomData, + } + } +} + +impl Clone for LayeredStep { + fn clone(&self) -> Self { + Self { + base: self.base.clone(), + _state: PhantomData, + } + } +} + +impl Hash for LayeredStep { + fn hash(&self, state: &mut H) { + self.base.hash(state); + } +} + +impl PartialEq for LayeredStep { + fn eq(&self, other: &Self) -> bool { + self.base.eq(&other.base) + } +} + +impl Eq for LayeredStep {} + +impl StepContainer for LayeredStep { + async fn container(self) -> Result<(Container, RunningLayeredStep)> { + Ok(( + Container::from_layer(&self.base).await?, + BuildStep { + base: (), + _state: PhantomData, + }, + )) + } +} + +impl StepCommiter for RunningLayeredStep { + async fn commit(self, container: Container) -> Result { + Ok(BuildStep { + base: LayerId::builder().container(container).build().await?, + _state: PhantomData, + }) + } +} + +impl StepBuilder for LayeredStep { + async fn run_build_step(self, layer: Layer) -> Result { + #[cached::cached(sync_writes = "by_key", key = "Layer", convert = "{ layer.clone() }")] + async fn inner(step: LayeredStep, layer: &Layer) -> Result { + let (container, step) = step.container().await?; + layer.run(&container).await?; + let step = step.commit(container).await?; + + Ok(step) + } + Ok(inner(self, &layer).await?.into()) + } + + async fn finalize(self) -> Result { + async move { Ok(self.base) }.await + } +} diff --git a/next/src/layer/build_step/squashed_step.rs b/next/src/layer/build_step/squashed_step.rs new file mode 100644 index 00000000..aacc927e --- /dev/null +++ b/next/src/layer/build_step/squashed_step.rs @@ -0,0 +1,83 @@ +use std::marker::PhantomData; + +use blue_build_utils::platform::Platform; +use miette::Result; + +use crate::layer::{ + BuildLayer, FromLayerReference, Layer, LayerId, StepBuilder, + build_step::{ + BuildStep, Container, ReadyBuildStep, StepCommiter, StepContainer, + states::{ReadyState, RunningSquashedState}, + }, +}; + +/// A `BuildStep` for squashed builds. +pub type SquashedStep = BuildStep; + +/// A `BuildStep` in a `RunningState` for squashed builds. +pub type RunningSquashedStep = BuildStep<(), RunningSquashedState>; + +#[bon::bon] +impl SquashedStep { + #[builder(finish_fn = "build")] + pub fn new_squashed( + image: impl AsRef, + platform: Option, + ) -> Result { + Ok(Self { + base: Container::from_image() + .image(image.as_ref()) + .maybe_platform(platform) + .build()?, + _state: PhantomData, + }) + } +} + +impl From> for SquashedStep { + fn from(value: Box) -> Self { + Self { + base: value.base, + _state: PhantomData, + } + } +} + +impl StepContainer for SquashedStep { + async fn container(self) -> Result<(Container, RunningSquashedStep)> { + async move { + Ok(( + self.base, + BuildStep { + base: (), + _state: PhantomData, + }, + )) + } + .await + } +} + +impl StepCommiter for RunningSquashedStep { + async fn commit(self, container: Container) -> Result { + async move { + Ok(BuildStep { + base: container, + _state: PhantomData, + }) + } + .await + } +} + +impl StepBuilder for SquashedStep { + async fn run_build_step(self, layer: Layer) -> Result { + let (container, step) = self.container().await?; + layer.run(&container).await?; + Ok(step.commit(container).await?.into()) + } + + async fn finalize(self) -> Result { + LayerId::builder().container(self.base).build().await + } +} diff --git a/next/src/layer/copy.rs b/next/src/layer/copy.rs new file mode 100644 index 00000000..1bfe2162 --- /dev/null +++ b/next/src/layer/copy.rs @@ -0,0 +1,110 @@ +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use blue_build_utils::cmd_out; +use miette::{IntoDiagnostic, Result}; + +use crate::{ + layer::{ + BuildLayer, FinalizeLayer, Layer, + build_step::{Container, ReadyBuildStep, StepBuilder}, + }, + path::ContextPath, +}; + +/// A `Layer` for copying files into a build container. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct CopyLayer { + parent: Layer, + from: Option, + source: ContextPath, + destination: PathBuf, +} + +#[bon::bon] +impl CopyLayer { + /// Builder for `CopyLayer`. + #[builder(derive(Into))] + pub fn new( + /// The parent `Layer` to copy files onto. + #[builder(into)] + parent: Layer, + + /// The `Layer` of a stage to copy files from. + #[builder(into)] + from: Option, + + /// The source to copy from. You must supply + /// a `NormalizedPath` and `RelativePath` to + /// create a guraunteed path that resides + /// within the build's context + /// + /// # Errors + /// Will error if the `ContextPath` fails to + /// resolve properly. + #[builder(with = |context: impl AsRef, path: impl AsRef| -> Result<_> { + Ok(ContextPath::builder() + .path(path.as_ref())? + .context_dir(context.as_ref())? + .build()) + })] + source: ContextPath, + + /// The location in the build to copy the files to. + #[builder(into)] + destination: PathBuf, + ) -> Arc { + Arc::new(Self { + parent, + from, + source, + destination, + }) + } + + pub(super) fn parent(&self) -> Layer { + self.parent.clone() + } +} + +impl BuildLayer for Arc { + async fn run(&self, container: &Container) -> Result<()> { + let from = match &self.from { + None => None, + Some(layer) => Some(layer.clone().finalize().await?), + }; + let container = container.as_str().to_owned(); + let source_path = self.source.path().to_owned(); + let source_context = self.source.context().to_owned(); + let destination = self.destination.clone(); + tokio::task::spawn_blocking(move || { + cmd_out!( + err_msg = format!( + "Failed to copy files {}from path {} to path {}", + from.as_ref().map_or_default(|from| format!("from container {from} ")), + source_path.display(), + destination.display(), + ); + "buildah", + "copy", + format!("--contextdir={}", source_context.display()), + if let Some(layer_id) = &from => format!("--from={layer_id}"), + &container, + &source_path, + &destination, + ) + }) + .await + .into_diagnostic()? + } + + async fn build(&self) -> Result { + self.parent + .build() + .await? + .run_build_step(self.clone().into()) + .await + } +} diff --git a/next/src/layer/from.rs b/next/src/layer/from.rs new file mode 100644 index 00000000..c2ec3ca8 --- /dev/null +++ b/next/src/layer/from.rs @@ -0,0 +1,175 @@ +use std::sync::Arc; + +use blue_build_utils::platform::Platform; +use miette::Result; + +use crate::{ + layer::{ + BuildLayer, LayerId, + build_step::{BuildStep, Container, LayeredStep, ReadyBuildStep}, + }, + reference::PinnedReference, +}; + +/// An enum to various types of +/// references that can be used to +/// start an image from. +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub enum FromLayerReference { + /// A scratch layer that has nothing in it. + #[default] + Scratch, + + /// A `PinnedReference` from an OCI or Docker registry. + Image(PinnedReference), + + /// A `LayerId` from another build. + Layer(LayerId), +} + +impl From for FromLayerReference { + fn from(value: PinnedReference) -> Self { + Self::Image(value) + } +} + +impl From for FromLayerReference { + fn from(value: LayerId) -> Self { + Self::Layer(value) + } +} + +impl AsRef for FromLayerReference { + fn as_ref(&self) -> &Self { + self + } +} + +impl std::fmt::Display for FromLayerReference { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::Scratch => "scratch".to_string(), + Self::Image(image) => format!("docker://{image}"), + Self::Layer(layer) => layer.to_string(), + } + ) + } +} + +/// The root `Layer` in the build graph. +/// +/// This object is needed to start creating +/// a build graph. It's possible to start +/// from "scratch", an image, or another build. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct FromLayer { + from: FromLayerReference, + platform: Option, + squash: bool, +} + +#[bon::bon] +impl FromLayer { + /// Builder for a `FromLayer`. + #[builder(derive(Into))] + pub fn new( + /// The starting reference for the build. + /// + /// If not defined, starts the build on + /// a "scratch" image. + #[builder(into, default)] + from: FromLayerReference, + + /// The platform to build for. + /// + /// Non-natvie platforms require emulation + /// in order to run. If not defined, uses the + /// native platform of the host. + platform: Option, + + /// Squash all changes of this build into + /// a single layer. + /// + /// Prevents caching build steps, but performs + /// better when building from inside a container. + #[builder(default)] + squash: bool, + ) -> Arc { + Arc::new(Self { + from, + platform, + squash, + }) + } + + pub(crate) fn platform(&self) -> Platform { + self.platform.unwrap_or_default() + } +} + +impl BuildLayer for Arc { + async fn run(&self, _container: &Container) -> Result<()> { + unimplemented!() + } + + async fn build(&self) -> Result { + #[cached::cached( + sync_writes = "by_key", + convert = "{ layer.clone() }", + key = "Arc" + )] + async fn inner(layer: &Arc) -> Result { + BuildStep::new_layered() + .image(&layer.from) + .maybe_platform(layer.platform) + .build() + .await + } + + Ok(if self.squash { + BuildStep::new_squashed() + .image(&self.from) + .maybe_platform(self.platform) + .build()? + .into() + } else { + inner(self).await?.into() + }) + } +} + +#[cfg(test)] +mod test { + + use blue_build_utils::platform::Platform; + use oci_client::Reference; + use rstest::rstest; + + use crate::{ + layer::{FinalizeLayer, from::FromLayer}, + reference::PinnedReference, + }; + + #[tokio::test] + #[rstest] + #[case("quay.io/fedora/fedora")] + #[case("docker.io/library/alpine")] + async fn eval(#[case] image: &str) { + let image: Reference = image.parse().unwrap(); + let image = PinnedReference::pin_image(image).await.unwrap(); + let platform = Platform::default(); + + let layer = FromLayer::builder().from(image).platform(platform).build(); + + dbg!(&layer); + + let id = layer.finalize().await; + + dbg!(&id); + + assert!(id.is_ok()); + } +} diff --git a/next/src/layer/mount.rs b/next/src/layer/mount.rs new file mode 100644 index 00000000..2ebb93b5 --- /dev/null +++ b/next/src/layer/mount.rs @@ -0,0 +1,267 @@ +use std::{fmt::Display, path::PathBuf, sync::Arc}; + +use lazy_regex::regex; +use miette::{Result, bail}; + +use crate::{ + layer::{FinalizeLayer, Layer, LayerId}, + path::ContextPath, +}; + +/// A `Mount` for a layer. Typically used in `RunLayer`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Mount { + typ: MountType, + common: MountCommon, +} + +#[bon::bon] +impl Mount { + /// Builder for `Mount`. + #[builder(finish_fn = "build", derive(Into))] + pub fn new_bind( + /// The source of the bind mount. + #[builder(into)] + source: MountSource, + + /// The destination in the build to mount to. + #[builder(into)] + destination: PathBuf, + + /// The SELinux mode to set on the mount. + #[builder(default)] + selinux: SeLinuxMode, + + /// The access mode of the mount. + #[builder(default)] + mode: MountMode, + ) -> Arc { + Arc::new(Self { + typ: MountType::Bind { source, selinux }, + common: MountCommon { destination, mode }, + }) + } + + #[builder(finish_fn = "build", derive(Into))] + pub fn new_cache( + /// The global `CacheId` for the mount. + /// + /// # Errors + /// Will fail if the cache ID has invalid characters. + #[builder(with = |id: &str| -> miette::Result<_> { + CacheId::try_from(id) + })] + id: CacheId, + + /// Source `PathBuf` in the cache mount. + #[builder(into)] + source: Option, + + /// The destination in the build to mount to. + #[builder(into)] + destination: PathBuf, + + /// The access mode of the mount. + #[builder(default)] + mode: MountMode, + ) -> Arc { + Arc::new(Self { + typ: MountType::Cache { id, source }, + common: MountCommon { destination, mode }, + }) + } +} + +impl Mount { + pub(super) async fn finalize(self: Arc) -> Result { + #[cached::cached] + async fn inner(mount: Arc) -> Result { + Ok(format!( + "--mount=type={mount}", + mount = match &mount.typ { + MountType::Bind { source, selinux } => { + let source = source.finalize().await?; + format!("bind{source}{common}{selinux}", common = mount.common) + } + MountType::Cache { id, source } => format!( + "cache,id={id}{src}{common}", + src = source + .as_ref() + .map_or_default(|source| source.display().to_string()), + common = mount.common + ), + } + )) + } + inner(self).await + } +} + +/// The type of the `Mount`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +enum MountType { + /// Bind mount. + Bind { + /// The source of the mount. + source: MountSource, + + /// The SELinux mode of the mount. + selinux: SeLinuxMode, + }, + /// Cache mount. + Cache { + /// the ID of the cache. + id: CacheId, + + /// The source path in the cache mount. + source: Option, + }, +} + +/// An ID for cache `Mount`s. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CacheId(String); + +impl TryFrom<&str> for CacheId { + type Error = miette::Error; + + fn try_from(value: &str) -> std::prelude::v1::Result { + if regex!("[a-zA-Z0-9_-]+").is_match(value) { + Ok(Self(value.to_string())) + } else { + bail!("String {value} a valid CacheId") + } + } +} + +impl std::fmt::Display for CacheId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// SELinux mode for the mount. +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub enum SeLinuxMode { + /// Blank label. + #[default] + Default, + + /// Labels the mount for shared access. + /// + /// Equivalent to `"z"`. + Shared, + + /// Labels the mount for private access. + /// + /// Equivalent to `"Z"`. + Private, +} + +impl Display for SeLinuxMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::Default => "", + Self::Shared => ",z", + Self::Private => ",Z", + } + ) + } +} + +/// Struct containing common `Mount` options. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct MountCommon { + /// The destination in the container to mount. + destination: PathBuf, + + /// The mode the mount is set to. + mode: MountMode, +} + +impl Display for MountCommon { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, ",dst={}{}", self.destination.display(), self.mode) + } +} + +/// The access mode of the `Mount`. +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +pub enum MountMode { + /// The default mode for the mount. + #[default] + Default, + + /// Set access for the mount to read only. + /// + /// Equivalent to `"ro"`. + ReadOnly, + + /// Set access for the mount to read write. + /// + /// Equivalent to `"rw"`. + ReadWrite, +} + +impl Display for MountMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::Default => "", + Self::ReadOnly => ",ro", + Self::ReadWrite => ",rw", + } + ) + } +} + +/// The source for the mount. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum MountSource { + /// A `Layer` source with a `PathBuf`. + Layer(Layer, PathBuf), + + /// A completed `Layer` in the form of a `LayerId`. + CompleteLayer(LayerId, PathBuf), + + /// A `ContextPath` mount. + Path(ContextPath), +} + +impl, P: Into> From<(L, P)> for MountSource { + fn from((layer, path): (L, P)) -> Self { + Self::Layer(layer.into(), path.into()) + } +} + +impl> From<(LayerId, P)> for MountSource { + fn from((layer, path): (LayerId, P)) -> Self { + Self::CompleteLayer(layer, path.into()) + } +} + +impl From for MountSource { + fn from(value: ContextPath) -> Self { + Self::Path(value) + } +} + +impl MountSource { + async fn finalize(&self) -> Result { + Ok(match self { + Self::Layer(layer, src) => { + let layer = layer.clone().finalize().await?; + format!(",from={layer},src={src}", src = src.display()) + } + Self::CompleteLayer(layer, src) => { + format!(",from={layer},src={src}", src = src.display()) + } + Self::Path(path) => format!(",src={}", path.path().display()), + }) + } +} diff --git a/next/src/layer/run.rs b/next/src/layer/run.rs new file mode 100644 index 00000000..69d7ad53 --- /dev/null +++ b/next/src/layer/run.rs @@ -0,0 +1,202 @@ +use std::{collections::BTreeMap, process::Command, sync::Arc}; + +use blue_build_utils::cmd_out; +use futures::future; +use miette::{IntoDiagnostic, Result}; +use tokio::task::JoinHandle; + +use crate::layer::{ + BuildLayer, Layer, LayerCommand, Mount, + build_step::{Container, ReadyBuildStep, StepBuilder}, +}; + +#[derive(Debug, PartialEq, Eq, Hash)] +pub enum RunCommand { + /// Sanitized command that helps + /// with escape sequences. + Sanitized(LayerCommand), + + /// Unsanitized command that is + /// passed directly into the shell. + Unsanitized(String), +} + +impl From for RunCommand { + fn from(value: Command) -> Self { + Self::Sanitized(value.into()) + } +} + +impl From for RunCommand { + fn from(value: LayerCommand) -> Self { + Self::Sanitized(value) + } +} + +impl From for RunCommand { + fn from(value: String) -> Self { + Self::Unsanitized(value) + } +} + +impl From<&str> for RunCommand { + fn from(value: &str) -> Self { + Self::from(value.to_string()) + } +} + +impl std::fmt::Display for RunCommand { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::Sanitized(cmd) => cmd.to_string(), + Self::Unsanitized(cmd) => cmd.clone(), + } + ) + } +} + +/// A `Layer` that allows you to run a `Command` +/// in a build container. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct RunLayer { + parent: Layer, + args: BTreeMap, + shell: LayerCommand, + command: RunCommand, + mounts: Vec>, +} + +#[bon::bon] +impl RunLayer { + /// Builder for a `RunLayer` + #[builder(derive(Into))] + pub fn new( + /// The `Layer` to run a command on. + #[builder(into)] + parent: Layer, + + /// The equivalent of `ARG` for this single instruction. + #[builder(default, with = FromIterator::from_iter)] + args: BTreeMap, + + /// The `Command` to run in the container. + #[builder(into)] + command: RunCommand, + + /// The shell `Command` to use for the run. + /// + /// The `cmd` will be passed in-full as a + /// single arg after all shell args. + #[builder( + into, + default = LayerCommand::from( + comlexr::cmd!("/bin/sh", "-c") + ) + )] + shell: LayerCommand, + + /// The `Mount`s to use during the run. + #[builder(default, with = FromIterator::from_iter)] + mounts: Vec>, + ) -> Arc { + Arc::new(Self { + parent, + args, + shell, + command, + mounts, + }) + } + + pub(super) fn parent(&self) -> Layer { + self.parent.clone() + } +} + +impl BuildLayer for Arc { + async fn run(&self, container: &Container) -> Result<()> { + let mounts = future::try_join_all(self.mounts.iter().map(build_mount)) + .await + .into_diagnostic()? + .into_iter() + .collect::>>()?; + + let platform = self.parent.get_root().platform(); + let shell_program = self.shell.program.clone(); + let shell_args = self.shell.args.clone(); + let args = self.args.clone(); + let command = self.command.to_string(); + let container = container.as_str().to_owned(); + + tokio::task::spawn_blocking(move || { + cmd_out!( + err_msg = format!("Failed to run command\n{command}"); + "buildah", + "run", + "--add-history", + "--tty=false", + format!("--env=TARGETARCH={platform}"), + for (key, value) in args => format!("--env={key}={value}"), + for mounts, + container, + "--", + shell_program, + for shell_args, + &command, + ) + }) + .await + .into_diagnostic()? + } + + async fn build(&self) -> Result { + self.parent + .build() + .await? + .run_build_step(self.clone().into()) + .await + } +} + +fn build_mount(mount: &Arc) -> JoinHandle> { + let mount = mount.clone(); + tokio::spawn(async { mount.finalize().await }) +} + +#[cfg(test)] +mod test { + use std::process::Command; + + use comlexr::cmd; + use rstest::rstest; + + use crate::{ + layer::{FinalizeLayer, from::FromLayer, run::RunLayer}, + reference::PinnedReference, + }; + + #[tokio::test] + #[rstest] + #[case( + cmd!("touch", "/test") + )] + async fn eval(#[case] cmd: Command) { + let image = PinnedReference::pin_image("quay.io/fedora/fedora".parse().unwrap()) + .await + .unwrap(); + let from_layer = FromLayer::builder().from(image).build(); + + let layer = RunLayer::builder().parent(from_layer).command(cmd).build(); + + dbg!(&layer); + + let id = layer.finalize().await; + + dbg!(&id); + + assert!(id.is_ok()); + } +} diff --git a/next/src/layer/unshare.rs b/next/src/layer/unshare.rs new file mode 100644 index 00000000..0ad53ac3 --- /dev/null +++ b/next/src/layer/unshare.rs @@ -0,0 +1,68 @@ +use std::sync::Arc; + +use blue_build_utils::cmd_out; +use miette::{IntoDiagnostic, Result}; + +use crate::layer::{BuildLayer, Layer, LayerCommand, build_step::ReadyBuildStep}; + +use super::{StepBuilder, build_step::Container}; + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct UnshareLayer { + parent: Layer, + commands: Vec, +} + +#[bon::bon] +impl UnshareLayer { + #[builder(derive(Into))] + pub fn new( + /// The `Layer` to mount and run host commands on. + #[builder(into)] + parent: Layer, + + /// The host commands to pass into `buildah unshare`. + #[builder(with = FromIterator::from_iter)] + commands: Vec, + ) -> Arc { + Arc::new(Self { parent, commands }) + } + + pub(super) fn parent(&self) -> Layer { + self.parent.clone() + } +} + +impl BuildLayer for Arc { + async fn run(&self, container: &Container) -> Result<()> { + let container = container.as_str().to_owned(); + let commands = self + .commands + .iter() + .map(ToString::to_string) + .collect::>() + .join(";\n"); + tokio::task::spawn_blocking(move || { + cmd_out!( + err_msg = format!("Failed to run commands in unshare:\n{commands}"); + "buildah", + "unshare", + format!("--mount=BB_UNSHARE_MOUNT={container}"), + "--", + "/bin/sh", + "-c", + &commands + ) + }) + .await + .into_diagnostic()? + } + + async fn build(&self) -> Result { + self.parent + .build() + .await? + .run_build_step(self.clone().into()) + .await + } +} diff --git a/next/src/lib.rs b/next/src/lib.rs new file mode 100644 index 00000000..ce49629d --- /dev/null +++ b/next/src/lib.rs @@ -0,0 +1,389 @@ +//! This library is the Next Generation Build Engine for BlueBuild. +//! +//! This build engine is based on `buildah` and allows building a +//! directed acyclical graph (DAG) from a BlueBuild recipe. This +//! engine takes advantage of OCI per-layer operations +//! that allow us evaluate the state of the image mid-build that a +//! standard `Containerfile` is not capable of. +//! +#![doc = include_str!("../LGPL_2.1.txt")] + +mod build_scripts; +pub mod layer; +pub mod path; +pub mod reference; + +use std::{collections::HashMap, iter::once, path::Path, process::Command, sync::Arc}; + +use blue_build_recipe::{ModuleRequiredFields, Recipe, RecipeGetters, StageRequiredFields}; +use blue_build_utils::{constants::NUSHELL_IMAGE, platform::Platform}; +use futures::{ + future::{self}, + stream::{self, TryStreamExt}, +}; +use miette::{IntoDiagnostic, Result, bail}; + +use crate::{ + layer::{CopyLayer, FinalizeLayer, FromLayer, Layer, LayerId, Mount, MountMode, RunLayer}, + reference::PinnedReference, +}; + +#[derive(Debug, Clone)] +pub struct Manifest { + images: HashMap, +} + +#[bon::bon] +impl Manifest { + #[builder] + pub async fn new( + recipe: &Recipe, + build_scripts_layer: &LayerId, + squash: bool, + context_dir: &Path, + ) -> Result { + let platforms = recipe.get_platforms(); + let create_image = async |platform| -> Result<(Platform, Image)> { + let image_plan = Image::builder() + .recipe(recipe) + .platform(platform) + .squash(squash) + .context_dir(context_dir) + .build_scripts_layer(build_scripts_layer) + .build() + .await?; + Ok((platform, image_plan)) + }; + + let images = future::try_join_all(platforms.iter().copied().map(create_image)) + .await? + .into_iter() + .collect(); + + Ok(Self { images }) + } + + /// Runs a build for all platforms + /// in an image. + /// + /// # Errors + /// Will error if any of the builds fail. + pub async fn build(self) -> Result> { + future::try_join_all(self.images.into_iter().map(|(platform, image)| { + tokio::spawn(async move { image.build().await.map(|layer| (platform, layer)) }) + })) + .await + .into_diagnostic()? + .into_iter() + .collect() + } +} + +#[derive(Debug, Clone)] +pub struct Image { + layers: Layer, +} + +#[bon::bon] +impl Image { + #[builder] + pub async fn new( + /// The recipe image to build. + recipe: &Recipe, + + /// The platform to build. + #[builder(default)] + platform: Platform, + + /// Squash the build into a single + /// layer on the base image. + #[builder(default)] + squash: bool, + + /// The layer containing the build scripts. + build_scripts_layer: &LayerId, + + /// The build's context directory. + context_dir: &Path, + ) -> Result { + let image = PinnedReference::pin_image(recipe.base_image_ref()?).await?; + let from = FromLayer::builder() + .from(image) + .platform(platform) + .squash(squash) + .build(); + let keys = RunLayer::builder() + .parent(from) + .mounts([keys_mnt(recipe, context_dir)?]) + .command("mkdir -p /etc/pki/containers/ && cp /tmp/keys/* /etc/pki/containers/") + .build(); + let stages = recipe + .get_processed_stages() + .into_iter() + .cloned() + .collect::>(); + let layers = stream::iter( + recipe + .get_processed_modules() + .into_iter() + .map(Ok::<_, miette::Error>), + ) + .try_fold(keys.into(), move |parent, module| { + let stages = stages.clone(); + async move { + add_module( + parent, + module, + recipe, + build_scripts_layer, + context_dir, + &stages, + &[], + ) + .await + } + }) + .await?; + Ok(Self { layers }) + } + + /// Build the image. + /// + /// # Errors + /// Will error if the build fails. + pub async fn build(self) -> Result { + self.layers.finalize().await + } +} + +async fn add_module<'a>( + parent: Layer, + module: &ModuleRequiredFields, + recipe: &'a Recipe, + build_scripts_layer: &'a LayerId, + context_dir: &'a Path, + stages: &[StageRequiredFields], + traversed_stages: &[String], +) -> Result { + Ok(match module.module_type.typ() { + "copy" => { + let Some(source) = module.config["src"].as_str() else { + bail!("The `src` property was expected on the copy module."); + }; + let Some(destination) = module.config["dest"].as_str() else { + bail!("The `dest` property was expected on the copy module."); + }; + let from = if let Some(stage) = module.config["from"] + .as_str() + .and_then(|from| stages.iter().find(|stage| stage.name == from)) + { + Some( + create_stage( + recipe, + build_scripts_layer, + context_dir, + stages, + traversed_stages, + stage, + ) + .await?, + ) + } else { + None + }; + CopyLayer::builder() + .parent(parent) + .maybe_from(from) + .source(context_dir, source)? + .destination(destination) + .build() + .into() + } + "containerfile" => panic!("The 'containerfile' type is not supported"), + _ => { + let mounts = [ + build_scripts_mnt(build_scripts_layer), + files_mnt(context_dir)?, + nushell_mnt().await?, + modules_mnt(context_dir, module).await?, + ]; + RunLayer::builder() + .parent(parent) + .args([ + ("CONFIG_DIRECTORY".to_string(), "/tmp/files".to_string()), + ("MODULE_DIRECTORY".to_string(), "/tmp/modules".to_string()), + ("IMAGE_NAME".to_string(), recipe.get_name().to_string()), + ( + "BASE_IMAGE".to_string(), + recipe.get_base_image().to_string(), + ), + ( + "IMAGE_REGISTRY".to_string(), + // TODO: replace with registry + "ghcr.io/blue-build/cli".to_string(), + ), + ("BB_BUILD_FEATURES".to_string(), String::new()), + ]) + .command(Command::try_from(module)?) + .mounts(mounts) + .build() + .into() + } + }) +} + +fn build_scripts_mnt(build_scripts_layer: &LayerId) -> Arc { + Mount::new_bind() + .source((build_scripts_layer.clone(), "/scripts")) + .destination("/tmp/scripts") + .build() +} + +async fn nushell_mnt() -> Result> { + let image = PinnedReference::pin_image( + format!("{NUSHELL_IMAGE}:default") + .parse() + .into_diagnostic()?, + ) + .await?; + Ok(Mount::new_bind() + .source((FromLayer::builder().from(image).build(), "/nu")) + .destination("/usr/libexec/bluebuild/nu") + .build()) +} + +fn files_mnt(context_dir: &Path) -> Result> { + Ok(Mount::new_bind() + .source(( + CopyLayer::builder() + .parent(FromLayer::builder().build()) + .source(context_dir, "./files")? + .destination("/files") + .build(), + "/files", + )) + .destination("/tmp/files") + .build()) +} + +fn keys_mnt(recipe: &Recipe, context_dir: &Path) -> Result> { + Ok(Mount::new_bind() + .source(( + CopyLayer::builder() + .parent(FromLayer::builder().build()) + .source(context_dir, "cosign.pub")? + .destination(format!("/keys/{}.pub", recipe.get_name().replace('/', "_"))) + .build(), + "/keys", + )) + .destination("/tmp/keys") + .build()) +} + +async fn modules_mnt(context_dir: &Path, module: &ModuleRequiredFields) -> Result> { + Ok(if let Some(source) = module.get_non_local_source() { + let source = PinnedReference::pin_image(source.parse().into_diagnostic()?).await?; + Mount::new_bind() + .source((FromLayer::builder().from(source).build(), "/modules")) + .destination("/tmp/modules") + .mode(MountMode::ReadWrite) + .build() + } else if module.is_local_source() { + Mount::new_bind() + .source(( + CopyLayer::builder() + .parent(FromLayer::builder().build()) + .source(context_dir, "./modules")? + .destination("/modules") + .build(), + "/modules", + )) + .destination("/tmp/modules") + .mode(MountMode::Default) + .build() + } else { + let image = + PinnedReference::pin_image(module.get_module_image().parse().into_diagnostic()?) + .await?; + Mount::new_bind() + .source((FromLayer::builder().from(image).build(), "/modules")) + .destination("/tmp/modules") + .mode(MountMode::ReadWrite) + .build() + }) +} + +async fn create_stage( + recipe: &Recipe, + build_scripts_layer: &LayerId, + context_dir: &Path, + stages: &[StageRequiredFields], + traversed_stages: &[String], + stage: &StageRequiredFields, +) -> Result { + if traversed_stages.contains(&stage.name) { + bail!("Hit cycle in build graph:\n{traversed_stages:?}"); + } + let traversed_stages = Arc::new( + once(stage.name.clone()) + .chain(traversed_stages.iter().cloned()) + .collect::>(), + ); + let image = PinnedReference::pin_image(stage.from.parse().into_diagnostic()?).await?; + let from = FromLayer::builder() + .from(image) + .maybe_platform(stage.platform) + .build(); + + stream::iter(stage.get_processed_modules().into_iter().map(Ok)) + .try_fold(from.into(), move |parent, module| { + let traversed_stages = traversed_stages.clone(); + async move { + // We pin the future since this is where the recursion starts. + Box::pin(add_module( + parent, + module, + recipe, + build_scripts_layer, + context_dir, + stages, + &traversed_stages, + )) + .await + } + }) + .await +} + +#[cfg(test)] +mod test { + use std::sync::LazyLock; + + use crate::build_scripts::build_scripts_layer; + + use super::*; + use rstest::rstest; + + static CONTEXT_DIR: LazyLock<&Path> = LazyLock::new(|| Path::new("./tests/repo/")); + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[rstest] + async fn image() { + let build_scripts_layer = build_scripts_layer().await.unwrap(); + let recipe = Recipe::builder() + .path("./tests/repo/recipes/recipe.yml") + .build() + .unwrap(); + let image = Image::builder() + .recipe(&recipe) + .context_dir(&CONTEXT_DIR) + .build_scripts_layer(&build_scripts_layer) + .build() + .await + .unwrap(); + + dbg!(&image); + + assert!(image.build().await.is_ok()); + } +} diff --git a/next/src/path.rs b/next/src/path.rs new file mode 100644 index 00000000..edba4445 --- /dev/null +++ b/next/src/path.rs @@ -0,0 +1,171 @@ +use std::{ + ops::Deref, + path::{Path, PathBuf}, +}; + +use miette::{IntoDiagnostic, bail}; + +/// A normalized path. This path is guaranteed +/// to be absolute and exist. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct NormalizedPath(PathBuf); + +impl TryFrom for NormalizedPath { + type Error = miette::Error; + + fn try_from(value: PathBuf) -> std::result::Result { + value.canonicalize().into_diagnostic().map(Self) + } +} + +impl TryFrom<&Path> for NormalizedPath { + type Error = miette::Error; + + fn try_from(value: &Path) -> std::result::Result { + value.canonicalize().into_diagnostic().map(Self) + } +} + +impl Deref for NormalizedPath { + type Target = Path; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl AsRef for NormalizedPath { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +/// An absolute path. This path is guaranteed +/// to be absolute. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct AbsolutePath(PathBuf); + +impl TryFrom for AbsolutePath { + type Error = miette::Error; + + fn try_from(value: PathBuf) -> Result { + if value.is_absolute() { + Ok(Self(value)) + } else { + bail!("Path {} is not absolute", value.display()) + } + } +} + +impl TryFrom<&Path> for AbsolutePath { + type Error = miette::Error; + + fn try_from(value: &Path) -> Result { + Self::try_from(value.to_path_buf()) + } +} + +impl Deref for AbsolutePath { + type Target = Path; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl AsRef for AbsolutePath { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +/// A relative path. This path is guaranteed +/// to be relative. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RelativePath(PathBuf); + +impl TryFrom for RelativePath { + type Error = miette::Error; + + fn try_from(value: PathBuf) -> Result { + if value.is_relative() { + Ok(Self(value)) + } else { + bail!("Path {} is not absolute", value.display()) + } + } +} + +impl TryFrom<&Path> for RelativePath { + type Error = miette::Error; + + fn try_from(value: &Path) -> Result { + Self::try_from(value.to_path_buf()) + } +} + +impl Deref for RelativePath { + type Target = Path; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl AsRef for RelativePath { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +/// A path to be used during a copy or mount. +/// Contains a `NormalizedPath` for the context +/// and a `RelativePath` to be used for a path +/// inside the context. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ContextPath(NormalizedPath, RelativePath); + +#[bon::bon] +impl ContextPath { + /// Create a `ContextPath` from a `NormalizedPath` + /// given a `RelativePath`. This verifies that the + /// path is within the context. + /// + /// # Errors + /// Will error when the path provided is not within + /// the `NormalizedPath`. + #[builder] + pub const fn new( + /// The context dir where the path should be located. + /// + /// # Errors + /// Will error when the path doesn't exist. + #[builder(with = |path: impl AsRef| -> miette::Result<_> { + NormalizedPath::try_from(path.as_ref()) + })] + context_dir: NormalizedPath, + + /// The path inside the context dir. + /// + /// # Errors + /// Will error when the path isn't relative. + #[builder(with = |path: impl AsRef| -> miette::Result<_> { + RelativePath::try_from(path.as_ref()) + })] + path: RelativePath, + ) -> Self { + Self(context_dir, path) + } + + /// The path to the context. + #[must_use] + pub fn context(&self) -> &Path { + &self.0 + } + + /// The path inside the context. + #[must_use] + pub fn path(&self) -> &Path { + &self.1 + } +} diff --git a/next/src/reference.rs b/next/src/reference.rs new file mode 100644 index 00000000..15e705de --- /dev/null +++ b/next/src/reference.rs @@ -0,0 +1,43 @@ +use std::{ops::Deref, sync::Arc}; + +use blue_build_process_management::drivers::{Driver, InspectDriver, opts::GetMetadataOpts}; +use miette::Result; +use oci_client::Reference; + +/// A `Reference` that is pinned to a digest. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PinnedReference(Arc); + +impl PinnedReference { + /// Pins a `Reference` to a digest + /// by inspecting the registry. + /// + /// # Errors + /// Will error if the call to the registry fails. + pub async fn pin_image(image: Reference) -> Result { + Ok(if image.digest().is_some() { + Self(Arc::new(image)) + } else { + let inspection = + Driver::get_metadata(GetMetadataOpts::builder().image(&image).build()).await?; + + Self(Arc::new( + image.clone_with_digest(inspection.digest().to_string()), + )) + }) + } +} + +impl Deref for PinnedReference { + type Target = Reference; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::fmt::Display for PinnedReference { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/next/tests/repo/.gitignore b/next/tests/repo/.gitignore new file mode 100644 index 00000000..23b42e13 --- /dev/null +++ b/next/tests/repo/.gitignore @@ -0,0 +1,3 @@ +/Containerfile +*.iso +*.iso-CHECKSUM diff --git a/next/tests/repo/LICENSE b/next/tests/repo/LICENSE new file mode 100644 index 00000000..1b59eee9 --- /dev/null +++ b/next/tests/repo/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024-2026 BlueBuild contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/next/tests/repo/README.md b/next/tests/repo/README.md new file mode 100644 index 00000000..4b3aa527 --- /dev/null +++ b/next/tests/repo/README.md @@ -0,0 +1,3 @@ +# Test repo for BlueBuild Next Gen Builder + +The keys at `cosign.pub` and `cosign.key` are publicly exposed and are not to be trusted. These keys are for testing purposes only. diff --git a/next/tests/repo/cosign.key b/next/tests/repo/cosign.key new file mode 100644 index 00000000..f5147df0 --- /dev/null +++ b/next/tests/repo/cosign.key @@ -0,0 +1,11 @@ +-----BEGIN ENCRYPTED SIGSTORE PRIVATE KEY----- +eyJrZGYiOnsibmFtZSI6InNjcnlwdCIsInBhcmFtcyI6eyJOIjo2NTUzNiwiciI6 +OCwicCI6MX0sInNhbHQiOiJ0Wnk3RDl6VVdFODNNSHFQOEVNdjhEcDNoY2tibHVH +RW5vdjVJdkttdkNRPSJ9LCJjaXBoZXIiOnsibmFtZSI6Im5hY2wvc2VjcmV0Ym94 +Iiwibm9uY2UiOiI3Q2xzZjZTenRTZG9ZQUFBVDZsa3FDUTAwTTVoS1NPaiJ9LCJj +aXBoZXJ0ZXh0IjoiUzNmejI1cFNNenkwYTlKYjhJdjNIeWFHVlFZMVJqY3BjNHhp +WWtkak9VZnZRa3FuRjBkMFB4Qm1TRlBzWFJWU0FKSHdhMTdFbmg0NS92aTFXRjg3 +KzErektLaDNzaXBqbkYvTmI0NmhHTVYza2JZRldYVVdmRGFPbVR5NjVxRVFLQUNR +WitQa1NENVpHVHI5UGQxdDdjdlNTUDM4aGtySExZVmlTekR2elVGTk9WcnJiWSsx +TUdFbGdIcnQ4bG4wcmdaYjAzMWhsMGJtTkE9PSJ9 +-----END ENCRYPTED SIGSTORE PRIVATE KEY----- diff --git a/next/tests/repo/cosign.pub b/next/tests/repo/cosign.pub new file mode 100644 index 00000000..382b9488 --- /dev/null +++ b/next/tests/repo/cosign.pub @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEt98ClXjjPubkSQ2rwyZe87ON2YaL +zySpZs3r2MwFMZYHfT2CKYd31RzYU/xPUCoCYUcC8ka/HW2Kl416J+joTA== +-----END PUBLIC KEY----- diff --git a/next/tests/repo/files/scripts/example.sh b/next/tests/repo/files/scripts/example.sh new file mode 100644 index 00000000..fdb2e042 --- /dev/null +++ b/next/tests/repo/files/scripts/example.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Tell this script to exit if there are any errors. +# You should have this in every custom script, to ensure that your completed +# builds actually ran successfully without any errors! +set -oue pipefail + +# Your code goes here. +echo 'This is an example shell script' +echo 'Scripts here will run during build if specified in recipe.yml' diff --git a/next/tests/repo/files/system/etc/.gitkeep b/next/tests/repo/files/system/etc/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/next/tests/repo/files/system/etc/.gitkeep @@ -0,0 +1 @@ + diff --git a/next/tests/repo/files/system/usr/.gitkeep b/next/tests/repo/files/system/usr/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/next/tests/repo/modules/.gitkeep b/next/tests/repo/modules/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/next/tests/repo/recipes/recipe.yml b/next/tests/repo/recipes/recipe.yml new file mode 100644 index 00000000..f1388d46 --- /dev/null +++ b/next/tests/repo/recipes/recipe.yml @@ -0,0 +1,45 @@ +--- +# yaml-language-server: $schema=https://schema.blue-build.org/recipe-v1.json +# image will be published to ghcr.io// +name: cli/test-next-gen +# description will be included in the image's metadata +description: Test recipe for the BlueBuild Next Gen Builder + +# the base image to build on top of (FROM) and the version tag to use +base-image: ghcr.io/ublue-os/silverblue-main +image-version: latest # You can pin to a specific version of Fedora as well + +# module configuration, executed in order +# you can include multiple instances of the same module +modules: + - type: files + files: + - source: system + destination: / # copies files/system/* (* means everything inside it) into your image's root folder / + + - type: dnf + repos: + copr: + - atim/starship + install: + packages: + - micro + - starship + remove: + packages: + # example: removing firefox (in favor of the flatpak) + # "firefox" is the main package, "firefox-langpacks" is a dependency + - firefox + - firefox-langpacks # also remove firefox dependency (not required for all packages, this is a special case) + + - type: default-flatpaks + configurations: + - notify: true # Send notification after install/uninstall is finished (true/false) + scope: system + # If no repo information is specified, Flathub will be used by default + install: # system flatpaks we want all users to have and not remove + - org.mozilla.firefox + - org.gnome.Loupe + - scope: user # Also add Flathub user repo, but no user packages + + - type: signing # this sets up the proper policy & signing files for signed images to work fully diff --git a/process/Cargo.toml b/process/Cargo.toml index 613166bd..8c6692a5 100644 --- a/process/Cargo.toml +++ b/process/Cargo.toml @@ -25,6 +25,7 @@ chrono.workspace = true clap = { workspace = true, features = ["derive", "env"] } colored.workspace = true comlexr.workspace = true +futures.workspace = true indicatif.workspace = true lazy-regex.workspace = true log.workspace = true diff --git a/process/drivers.rs b/process/drivers.rs index 1189c36a..082f2352 100644 --- a/process/drivers.rs +++ b/process/drivers.rs @@ -32,7 +32,7 @@ use miette::{Context, Result, bail}; use oci_client::Reference; use uuid::Uuid; -use crate::{logging::Logger, signal_handler::DetachedContainer}; +use crate::{ASYNC_RUNTIME, logging::Logger, signal_handler::DetachedContainer}; use opts::{ BuildChunkedOciOpts, BuildOpts, BuildRechunkTagPushOpts, BuildTagPushOpts, CheckKeyPairOpts, ContainerOpts, CopyOciOpts, CreateContainerOpts, GenerateImageNameOpts, GenerateKeyPairOpts, @@ -227,7 +227,10 @@ impl Driver { info!("Retrieving OS version from {oci_ref}"); - let os_version = Self::get_metadata(GetMetadataOpts::builder().image(oci_ref).build()) + let os_version = ASYNC_RUNTIME + .block_on(Self::get_metadata( + GetMetadataOpts::builder().image(oci_ref).build(), + )) .and_then(|inspection| { trace!("{inspection:?}"); inspection.get_version().wrap_err_with(|| { @@ -441,8 +444,8 @@ impl SigningDriver for Driver { } impl InspectDriver for Driver { - fn get_metadata(opts: GetMetadataOpts) -> Result { - OciClientDriver::get_metadata(opts) + async fn get_metadata(opts: GetMetadataOpts<'_>) -> Result { + OciClientDriver::get_metadata(opts).await } } diff --git a/process/drivers/oci_client_driver.rs b/process/drivers/oci_client_driver.rs index 3a640cc7..a5bfb577 100644 --- a/process/drivers/oci_client_driver.rs +++ b/process/drivers/oci_client_driver.rs @@ -1,12 +1,13 @@ use blue_build_utils::credentials::Credentials; use cached::cached; +use futures::future; use log::{debug, trace}; use miette::{Context, IntoDiagnostic, Result}; use oci_client::{Reference, client::ClientConfig, manifest::OciManifest, secrets::RegistryAuth}; -use crate::{ - ASYNC_RUNTIME, - drivers::{InspectDriver, types::ImageMetadata}, +use crate::drivers::{ + InspectDriver, + types::{ImageConfig, ImageMetadata}, }; use super::opts::GetMetadataOpts; @@ -14,9 +15,9 @@ use super::opts::GetMetadataOpts; pub struct OciClientDriver; impl InspectDriver for OciClientDriver { - fn get_metadata(opts: GetMetadataOpts) -> Result { + async fn get_metadata(opts: GetMetadataOpts<'_>) -> Result { #[cached(key = "String", convert = r"{image.to_string()}")] - fn inner(image: &Reference) -> Result { + async fn inner(image: &Reference) -> Result { let client = oci_client::Client::new(ClientConfig::default()); let auth = match Credentials::get(image.registry()) { Some(Credentials::Basic { username, password }) => { @@ -33,8 +34,9 @@ impl InspectDriver for OciClientDriver { } }; - let (manifest, digest) = ASYNC_RUNTIME - .block_on(client.pull_manifest(image, &auth)) + let (manifest, digest) = client + .pull_manifest(image, &auth) + .await .into_diagnostic() .wrap_err_with(|| format!("Failed to pull the manifest for {image}"))?; debug!("Found OciManifest for {image}"); @@ -56,52 +58,7 @@ impl InspectDriver for OciClientDriver { trace!("Found digests: {manifest_digests:#?}"); - let configs = manifest_digests - .into_iter() - .map(|digest| { - let image = &image.clone_with_digest(digest.clone()); - let (image_manifest, image_manifest_digest) = ASYNC_RUNTIME - .block_on(client.pull_image_manifest(image, &auth)) - .into_diagnostic() - .wrap_err_with(|| format!("Failed to pull image manifest for {image}"))?; - debug!("Pulled image manifest for {image}"); - trace!("digest: {image_manifest_digest}"); - trace!("{image_manifest:#?}"); - - let config = { - let capacity = image_manifest.config.size; - let mut c: Vec = Vec::with_capacity( - capacity.try_into().into_diagnostic().wrap_err_with(|| { - format!( - concat!( - "Size of image {image} config ", - "({capacity}) could not be converted to usize" - ), - image = image, - capacity = capacity - ) - })?, - ); - ASYNC_RUNTIME - .block_on(client.pull_blob(image, &image_manifest.config, &mut c)) - .into_diagnostic() - .wrap_err_with(|| format!("Failed to pull blob for {image}"))?; - c - }; - Ok(( - image_manifest.config.digest, - serde_json::from_slice(&config) - .inspect(|config| trace!("{config:#?}")) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "Failed to convert config for {image} to ImageConfig:\n{}", - String::from_utf8_lossy(&config) - ) - })?, - )) - }) - .collect::>>()?; + let configs = get_configs(manifest_digests, &client, &auth, image).await?; debug!("Retrieved configs for {image}"); trace!( @@ -118,9 +75,65 @@ impl InspectDriver for OciClientDriver { trace!("OciClientDriver::get_metadata({opts:?})"); if opts.no_cache { - inner_prime_cache(opts.image) + inner_prime_cache(opts.image).await } else { - inner(opts.image) + inner(opts.image).await } } } + +async fn get_configs( + manifest_digests: Vec<&String>, + client: &oci_client::Client, + auth: &RegistryAuth, + image: &Reference, +) -> Result> { + future::try_join_all(manifest_digests.into_iter().map( + async |digest| -> Result<(String, ImageConfig)> { + let image = &image.clone_with_digest(digest.clone()); + let (image_manifest, image_manifest_digest) = client + .pull_image_manifest(image, auth) + .await + .into_diagnostic() + .wrap_err_with(|| format!("Failed to pull image manifest for {image}"))?; + debug!("Pulled image manifest for {image}"); + trace!("digest: {image_manifest_digest}"); + trace!("{image_manifest:#?}"); + + let config = { + let capacity = image_manifest.config.size; + let mut c: Vec = Vec::with_capacity( + capacity.try_into().into_diagnostic().wrap_err_with(|| { + format!( + concat!( + "Size of image {image} config ", + "({capacity}) could not be converted to usize" + ), + image = image, + capacity = capacity + ) + })?, + ); + client + .pull_blob(image, &image_manifest.config, &mut c) + .await + .into_diagnostic() + .wrap_err_with(|| format!("Failed to pull blob for {image}"))?; + c + }; + Ok(( + image_manifest.config.digest, + serde_json::from_slice(&config) + .inspect(|config| trace!("{config:#?}")) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "Failed to convert config for {image} to ImageConfig:\n{}", + String::from_utf8_lossy(&config) + ) + })?, + )) + }, + )) + .await +} diff --git a/process/drivers/traits.rs b/process/drivers/traits.rs index 6e2cccbe..98372bcd 100644 --- a/process/drivers/traits.rs +++ b/process/drivers/traits.rs @@ -41,7 +41,8 @@ use super::{ }, }; use crate::{ - drivers::opts::PrivateKey, logging::CommandLogging, signal_handler::DetachedContainer, + ASYNC_RUNTIME, drivers::opts::PrivateKey, logging::CommandLogging, + signal_handler::DetachedContainer, }; trait PrivateDriver {} @@ -301,7 +302,7 @@ pub trait InspectDriver: PrivateDriver { /// /// # Errors /// Will error if it is unable to get the labels. - fn get_metadata(opts: GetMetadataOpts) -> Result; + fn get_metadata(opts: GetMetadataOpts) -> impl Future> + Send; } /// Allows agnostic running of containers. @@ -992,12 +993,12 @@ pub trait SigningDriver: PrivateDriver { .map_or_else(|| PathBuf::from("."), |d| d.to_path_buf()); let cosign_file_path = path.join(COSIGN_PUB_PATH); - let metadata = Driver::get_metadata( + let metadata = ASYNC_RUNTIME.block_on(Driver::get_metadata( GetMetadataOpts::builder() .image(opts.image) .no_cache(true) .build(), - )?; + ))?; debug!("Recieved metadata"); trace!("{metadata:#?}"); diff --git a/recipe/Cargo.toml b/recipe/Cargo.toml index 2af9b73d..5a8b9597 100644 --- a/recipe/Cargo.toml +++ b/recipe/Cargo.toml @@ -13,11 +13,13 @@ blue-build-utils = { version = "=0.9.37", path = "../utils" } cached.workspace = true colored.workspace = true +comlexr.workspace = true log.workspace = true miette.workspace = true oci-client.workspace = true indexmap.workspace = true serde.workspace = true +serde_json.workspace = true serde_yaml.workspace = true structstruck.workspace = true bon.workspace = true diff --git a/recipe/src/lib.rs b/recipe/src/lib.rs index cbfda8d5..af62e8b7 100644 --- a/recipe/src/lib.rs +++ b/recipe/src/lib.rs @@ -8,7 +8,7 @@ use std::{ use blue_build_utils::{ constants::{CONFIG_PATH, RECIPE_PATH}, container::Tag, - platform::Platform, + platform::{Platform, PlatformList}, secret::Secret, }; use cached::cached; @@ -55,8 +55,8 @@ trait RecipeSetters: RecipeGetters { } fn set_modules(&mut self, modules: Vec); - fn set_stages(&mut self, stages: Vec); + fn set_platforms(&mut self, platforms: PlatformList); } pub trait RecipeGetters { @@ -66,7 +66,7 @@ pub trait RecipeGetters { fn get_stages(&self) -> &[Stage]; fn get_labels(&self) -> HashMap<&str, &str>; fn get_alt_tags(&self) -> Option<&[Tag]>; - fn get_platforms(&self) -> &[Platform]; + fn get_platforms(&self) -> PlatformList; fn get_base_image(&self) -> Cow<'_, str>; fn get_bluebuild_version(&self) -> Option; fn get_cosign_version(&self) -> Option; @@ -173,14 +173,19 @@ pub enum Recipe { V2(Box), } +#[bon::bon] impl Recipe { /// Parse a recipe file /// /// # Errors /// Errors when a yaml file cannot be deserialized, /// or a linked module yaml file does not exist. - pub fn parse>(path: P) -> Result { - #[cached(key = "PathBuf", convert = r"{ path.into() }")] + #[builder] + pub fn new

(path: P, platforms: Option<&[Platform]>) -> Result + where + P: AsRef, + { + #[cached(key = "(PathBuf)", convert = r"{ path.into() }")] fn inner(path: &Path) -> Result { trace!("Recipe::parse({})", path.display()); @@ -204,9 +209,19 @@ impl Recipe { .into_diagnostic() .wrap_err_with(|| format!("Failed to parse recipe file {}", file_path.display()))?; recipe.process_from_files()?; + Ok(recipe) } - inner(path.as_ref()) + + let mut recipe = inner(path.as_ref())?; + + if let Some(platforms) = platforms + && !platforms.is_empty() + { + recipe.set_platforms(platforms.into()); + } + + Ok(recipe) } #[must_use] @@ -275,6 +290,19 @@ impl Serialize for Recipe { } } +impl Default for Recipe { + fn default() -> Self { + #[cfg(feature = "recipe-v2")] + { + Self::V2(RecipeV2::default().into()) + } + #[cfg(not(feature = "recipe-v2"))] + { + Self::V1(RecipeV1::default().into()) + } + } +} + macro_rules! impl_recipe { ($self:ident, $func:ident($($args:expr),*)) => { match $self { @@ -285,79 +313,47 @@ macro_rules! impl_recipe { }; } -impl RecipeGetters for Recipe { - fn get_name(&self) -> &str { - impl_recipe!(self, get_name()) - } - - fn get_description(&self) -> Option<&str> { - impl_recipe!(self, get_description()) - } - - fn get_modules(&self) -> &[Module] { - impl_recipe!(self, get_modules()) - } - - fn get_stages(&self) -> &[Stage] { - impl_recipe!(self, get_stages()) - } - - fn get_labels(&self) -> HashMap<&str, &str> { - impl_recipe!(self, get_labels()) - } - - fn base_image_ref(&self) -> Result { - impl_recipe!(self, base_image_ref()) - } - - fn get_alt_tags(&self) -> Option<&[Tag]> { - impl_recipe!(self, get_alt_tags()) - } - - fn get_platforms(&self) -> &[Platform] { - impl_recipe!(self, get_platforms()) - } - - fn get_base_image(&self) -> Cow<'_, str> { - impl_recipe!(self, get_base_image()) - } - - fn get_bluebuild_version(&self) -> Option { - impl_recipe!(self, get_bluebuild_version()) - } - - fn get_cosign_version(&self) -> Option { - impl_recipe!(self, get_cosign_version()) - } - - fn get_nushell_version(&self) -> Option { - impl_recipe!(self, get_nushell_version()) - } -} - -impl RecipeSetters for Recipe { - fn set_modules(&mut self, modules: Vec) { - impl_recipe!(self, set_modules(modules)); - } - - fn set_stages(&mut self, stages: Vec) { - impl_recipe!(self, set_stages(stages)); - } +macro_rules! getters { + ($($fun:ident -> $out:ty),*$(,)?) => { + impl RecipeGetters for Recipe { + $(fn $fun(&self) -> $out { + impl_recipe!(self, $fun()) + })* + } + }; } -impl Default for Recipe { - fn default() -> Self { - #[cfg(feature = "recipe-v2")] - { - Self::V2(RecipeV2::default().into()) - } - #[cfg(not(feature = "recipe-v2"))] - { - Self::V1(RecipeV1::default().into()) +macro_rules! setters { + ($($fun:ident($out:ty)),* $(,)*) => { + impl RecipeSetters for Recipe { + $(fn $fun(&mut self, value: $out) { + impl_recipe!(self, $fun(value)) + })* } - } + }; } +getters!( + get_name -> &str, + get_description -> Option<&str>, + get_modules -> &[Module], + get_stages -> &[Stage], + get_labels -> HashMap<&str, &str>, + base_image_ref -> Result, + get_alt_tags -> Option<&[Tag]>, + get_platforms -> PlatformList, + get_base_image -> Cow<'_, str>, + get_bluebuild_version -> Option, + get_cosign_version -> Option, + get_nushell_version -> Option, +); + +setters!( + set_modules(Vec), + set_stages(Vec), + set_platforms(PlatformList), +); + pub(crate) fn base_recipe_path() -> PathBuf { #[cfg(not(test))] let (legacy_path, recipe_path) = (PathBuf::from(CONFIG_PATH), PathBuf::from(RECIPE_PATH)); @@ -391,7 +387,7 @@ mod test { #[cfg_attr(feature = "recipe-v2", case::recipe_v2("recipes/recipe-v2.yml"))] fn parse_recipe(#[case] recipe_path: &str) { // serialize - let recipe = Recipe::parse(recipe_path).unwrap(); + let recipe = Recipe::builder().path(recipe_path).build().unwrap(); // deserialize serde_yaml::to_string(&recipe).unwrap(); diff --git a/recipe/src/module.rs b/recipe/src/module.rs index cee9ba95..5bd30038 100644 --- a/recipe/src/module.rs +++ b/recipe/src/module.rs @@ -1,13 +1,14 @@ -use std::path::PathBuf; +use std::{path::PathBuf, process::Command}; use blue_build_utils::{ constants::BLUE_BUILD_MODULE_IMAGE_REF, secret::Secret, syntax_highlighting::highlight_ser, }; use bon::Builder; use colored::Colorize; +use comlexr::{cmd, cmd_mut}; use indexmap::IndexMap; use log::trace; -use miette::{Result, bail}; +use miette::{IntoDiagnostic, Result, bail}; use serde::{Deserialize, Serialize}; use serde_yaml::Value; @@ -214,6 +215,27 @@ impl ModuleRequiredFields { } } +impl TryFrom<&ModuleRequiredFields> for Command { + type Error = miette::Report; + + fn try_from(value: &ModuleRequiredFields) -> Result { + let mut c = cmd!( + "/tmp/scripts/run_module.sh", + value.module_type.typ(), + serde_json::to_string(value).into_diagnostic()?, + ); + for (key, value) in value.get_env() { + cmd_mut!( + env { + key: value, + }; + &mut c + ); + } + Ok(c) + } +} + #[derive(Serialize, Deserialize, Debug, Clone, Builder, Default)] pub struct Module { #[serde(flatten, skip_serializing_if = "Option::is_none")] diff --git a/recipe/src/recipe_v1.rs b/recipe/src/recipe_v1.rs index 20057ad3..033825d3 100644 --- a/recipe/src/recipe_v1.rs +++ b/recipe/src/recipe_v1.rs @@ -1,7 +1,10 @@ use std::{borrow::Cow, collections::HashMap}; use blue_build_utils::{ - constants::COSIGN_IMAGE_VERSION, container::Tag, env_str::EnvString, platform::Platform, + constants::COSIGN_IMAGE_VERSION, + container::Tag, + env_str::EnvString, + platform::{Platform, PlatformList}, }; use bon::Builder; use miette::{Context, IntoDiagnostic, Result}; @@ -60,8 +63,9 @@ pub struct RecipeV1 { pub nushell_version: Option, /// The platforms to build for the image. - #[serde(skip_serializing_if = "Option::is_none")] - pub platforms: Option>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[builder(default)] + pub platforms: Vec, /// The version of cosign to install. #[serde(skip_serializing_if = "Option::is_none", rename = "cosign-version")] @@ -130,8 +134,8 @@ impl RecipeGetters for RecipeV1 { self.alt_tags.as_deref() } - fn get_platforms(&self) -> &[Platform] { - self.platforms.as_deref().unwrap_or(&[]) + fn get_platforms(&self) -> PlatformList { + PlatformList::from(&self.platforms) } fn get_bluebuild_version(&self) -> Option { @@ -171,6 +175,10 @@ impl RecipeSetters for RecipeV1 { self.stages_ext = Some(StagesExt::builder().stages(stages).build()); } } + + fn set_platforms(&mut self, platforms: PlatformList) { + self.platforms = platforms.into(); + } } #[cfg(test)] diff --git a/recipe/src/recipe_v2.rs b/recipe/src/recipe_v2.rs index 110d4be2..328e2a13 100644 --- a/recipe/src/recipe_v2.rs +++ b/recipe/src/recipe_v2.rs @@ -1,7 +1,10 @@ use std::{borrow::Cow, collections::HashMap, ops::Deref}; use blue_build_utils::{ - constants::BLUE_BUILD_DEFAULT_IMAGE, container::Tag, env_str::EnvString, platform::Platform, + constants::BLUE_BUILD_DEFAULT_IMAGE, + container::Tag, + env_str::EnvString, + platform::{Platform, PlatformList}, }; use bon::Builder; use miette::{Context, IntoDiagnostic}; @@ -227,19 +230,19 @@ impl From for RecipeV2 { }, }); match (value.platforms, has_versions) { - (None, false) => None, - (Some(platforms), false) => Some(RecipeV2Spec { + (platforms, false) if platforms.is_empty() => None, + (platforms, true) if platforms.is_empty() => Some(RecipeV2Spec { + platforms: Vec::new(), + tool_versions, + }), + (platforms, false) => Some(RecipeV2Spec { platforms, tool_versions: None, }), - (Some(platforms), true) => Some(RecipeV2Spec { + (platforms, true) => Some(RecipeV2Spec { platforms, tool_versions, }), - (None, true) => Some(RecipeV2Spec { - platforms: Vec::new(), - tool_versions, - }), } }, stages_ext: value.stages_ext, @@ -287,8 +290,10 @@ impl RecipeGetters for RecipeV2 { } } - fn get_platforms(&self) -> &[Platform] { - self.spec.as_ref().map_or(&[], |spec| &spec.platforms) + fn get_platforms(&self) -> PlatformList { + self.spec + .as_ref() + .map_or_default(|spec| PlatformList::from(&spec.platforms)) } fn get_base_image(&self) -> Cow<'_, str> { @@ -384,4 +389,15 @@ impl RecipeSetters for RecipeV2 { self.stages_ext = Some(StagesExt::builder().stages(stages).build()); } } + + fn set_platforms(&mut self, platforms: PlatformList) { + if let Some(spec) = &mut self.spec { + spec.platforms = platforms.into(); + } else { + self.spec = Some(RecipeV2Spec { + platforms: platforms.into(), + tool_versions: None, + }); + } + } } diff --git a/src/commands/bug_report.rs b/src/commands/bug_report.rs index a8c10bfc..604ebe3e 100644 --- a/src/commands/bug_report.rs +++ b/src/commands/bug_report.rs @@ -131,7 +131,7 @@ impl BugReportCommand { }) }); - Recipe::parse(&recipe_path).ok() + Recipe::builder().path(&recipe_path).build().ok() } } diff --git a/src/commands/build.rs b/src/commands/build.rs index ba2097a0..15070d54 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -5,6 +5,7 @@ use std::{ }; use blue_build_process_management::{ + ASYNC_RUNTIME, drivers::{ BuildChunkedOciDriver, BuildDriver, CiDriver, Driver, DriverArgs, InspectDriver, PostBuildDriver, RechunkDriver, SigningDriver, @@ -312,7 +313,10 @@ impl BuildCommand { containerfile.display() ); - let recipe = &Recipe::parse(recipe_path)?; + let recipe = &Recipe::builder() + .path(recipe_path) + .platforms(&self.platform) + .build()?; let tags = &Driver::generate_tags( GenerateTagsOpts::builder() .oci_ref(&recipe.base_image_ref()?) @@ -353,8 +357,6 @@ impl BuildCommand { }); let cache_image = cache_image.as_ref(); - let platforms = &platforms(&self.platform, recipe.get_platforms()); - let secrets = &recipe.get_secrets(); let image_ref = self.archive.as_ref().map_or_else( @@ -369,17 +371,20 @@ impl BuildCommand { ); let base_image = recipe.base_image_ref()?; - let base_digest = - Driver::get_metadata(GetMetadataOpts::builder().image(&base_image).build())? - .digest() - .to_owned(); + let base_digest = ASYNC_RUNTIME + .block_on(Driver::get_metadata( + GetMetadataOpts::builder().image(&base_image).build(), + ))? + .digest() + .to_owned(); let base_image_with_digest = base_image.clone_with_digest(base_digest); + let platforms = recipe.get_platforms(); let build_tag_opts = BuildTagPushOpts::builder() .image(&image_ref) .base_image(&base_image_with_digest) .containerfile(containerfile) - .platform(platforms) + .platform(&platforms) .squash(self.squash) .maybe_cache_from(cache_image) .maybe_cache_to(cache_image) @@ -429,10 +434,12 @@ impl BuildCommand { )? } else if self.build_chunked_oci { let base_image = recipe.base_image_ref()?; - let base_digest = - Driver::get_metadata(GetMetadataOpts::builder().image(&base_image).build())? - .digest() - .to_owned(); + let base_digest = ASYNC_RUNTIME + .block_on(Driver::get_metadata( + GetMetadataOpts::builder().image(&base_image).build(), + ))? + .digest() + .to_owned(); let remove_base_image = self .remove_base_image .then_some(base_image.clone_with_digest(base_digest)); @@ -449,7 +456,7 @@ impl BuildCommand { .build(), )? } else if self.rechunk { - self.rechunk(containerfile, recipe, tags, image, cache_image, platforms)? + self.rechunk(containerfile, recipe, tags, image, cache_image)? } else { Driver::build_tag_push(opts)? }; @@ -460,7 +467,7 @@ impl BuildCommand { .image(image) .retry_push(self.retry_push) .retry_count(self.retry_count) - .platforms(platforms) + .platforms(&recipe.get_platforms()) .build(), )?; } @@ -475,16 +482,16 @@ impl BuildCommand { tags: &[Tag], image_name: &Reference, cache_image: Option<&Reference>, - platforms: &[Platform], ) -> Result, miette::Error> { trace!( - "BuildCommand::rechunk({}, {recipe:?}, {tags:?}, {image_name}, {cache_image:?}, {platforms:?})", + "BuildCommand::rechunk({}, {recipe:?}, {tags:?}, {image_name}, {cache_image:?})", containerfile.display() ); let base_image = recipe.base_image_ref()?; - let base_digest = - &Driver::get_metadata(GetMetadataOpts::builder().image(&base_image).build())?; + let base_digest = &ASYNC_RUNTIME.block_on(Driver::get_metadata( + GetMetadataOpts::builder().image(&base_image).build(), + ))?; let base_digest = base_digest.digest(); let default_labels = generate_default_labels(recipe)?; @@ -494,7 +501,7 @@ impl BuildCommand { RechunkOpts::builder() .image(image_name) .containerfile(containerfile) - .platform(platforms) + .platform(&recipe.get_platforms()) .tags(tags) .push(self.push) .version(&format!( @@ -522,19 +529,6 @@ impl BuildCommand { } } -fn platforms(cli_platforms: &[Platform], recipe_platforms: &[Platform]) -> Vec { - let platforms = match (cli_platforms, recipe_platforms) { - ([], []) => vec![Platform::default()], - ([], recipe) => recipe.to_vec(), - (cli, _) => cli.to_vec(), - }; - assert!( - platforms.is_empty().not(), - "At least one platform must be built" - ); - platforms -} - #[cfg(test)] mod test { use blue_build_utils::platform::Platform; @@ -586,6 +580,9 @@ mod test { #[case] recipe_plat: &[Platform], #[case] expected: &[Platform], ) { - pretty_assertions::assert_eq!(&*super::platforms(cli_plat, recipe_plat), expected); + pretty_assertions::assert_eq!( + &*blue_build_utils::platforms(cli_plat, recipe_plat), + expected + ); } } diff --git a/src/commands/generate.rs b/src/commands/generate.rs index 962dc0cb..338a3aba 100644 --- a/src/commands/generate.rs +++ b/src/commands/generate.rs @@ -5,8 +5,9 @@ use std::{ }; use crate::{BuildScripts, DriverTemplate, commands::validate::ValidateCommand}; -use blue_build_process_management::drivers::{ - CiDriver, Driver, DriverArgs, InspectDriver, opts::GetMetadataOpts, +use blue_build_process_management::{ + ASYNC_RUNTIME, + drivers::{CiDriver, Driver, DriverArgs, InspectDriver, opts::GetMetadataOpts}, }; use blue_build_recipe::{Recipe, RecipeGetters}; use blue_build_template::{ContainerFileTemplate, Template}; @@ -123,7 +124,7 @@ impl GenerateCommand { }; debug!("Deserializing recipe"); - let recipe = Recipe::parse(&recipe_path)?; + let recipe = Recipe::builder().path(&recipe_path).build()?; trace!("recipe_de: {recipe:#?}"); if self.display_full_recipe { @@ -139,8 +140,9 @@ impl GenerateCommand { info!("Templating for recipe at {}", recipe_path.display()); let base_image = recipe.base_image_ref()?; - let base_digest = - &Driver::get_metadata(GetMetadataOpts::builder().image(&base_image).build())?; + let base_digest = &ASYNC_RUNTIME.block_on(Driver::get_metadata( + GetMetadataOpts::builder().image(&base_image).build(), + ))?; let base_digest = base_digest.digest(); let build_features = &[ #[cfg(feature = "bootc")] @@ -217,8 +219,9 @@ pub fn generate_default_labels(recipe: &Recipe) -> Result { - let recipe = Recipe::parse(recipe)?; + let recipe = Recipe::builder().path(recipe).build()?; args.extend([ format!( diff --git a/src/commands/switch.rs b/src/commands/switch.rs index 955ca4c4..53074548 100644 --- a/src/commands/switch.rs +++ b/src/commands/switch.rs @@ -62,7 +62,7 @@ impl BlueBuildCommand for SwitchCommand { bail!("There is a transaction in progress. Please cancel it using `rpm-ostree cancel`"); } - let recipe = Recipe::parse(&self.recipe)?; + let recipe = Recipe::builder().path(&self.recipe).build()?; let image_name = Driver::generate_image_name( GenerateImageNameOpts::builder() .name(recipe.get_name().trim()) diff --git a/utils/src/lib.rs b/utils/src/lib.rs index eef0f7eb..51e05628 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -32,7 +32,7 @@ use log::{trace, warn}; use miette::{Context, IntoDiagnostic, Result, miette}; use uuid::Uuid; -use crate::constants::CONTAINER_FILE; +use crate::{constants::CONTAINER_FILE, platform::Platform}; pub use command_output::*; @@ -185,3 +185,24 @@ pub fn tempdir_in>(dir: P) -> Result { .tempdir_in(dir) .into_diagnostic() } + +/// Get the list of platforms given +/// the platforms in the recipe and an +/// array of overrides. +/// +/// # Panics +/// Panics if no platforms are returned. At least +/// one platform is always expected. +#[must_use] +pub fn platforms(override_platforms: &[Platform], recipe_platforms: &[Platform]) -> Vec { + let platforms = match (override_platforms, recipe_platforms) { + ([], []) => vec![Platform::default()], + ([], recipe) => recipe.to_vec(), + (cli, _) => cli.to_vec(), + }; + assert!( + platforms.is_empty().not(), + "At least one platform must be built" + ); + platforms +} diff --git a/utils/src/macros.rs b/utils/src/macros.rs index f327a5b9..94eccd89 100644 --- a/utils/src/macros.rs +++ b/utils/src/macros.rs @@ -191,3 +191,110 @@ macro_rules! sudo_cmd { } }; } + +#[macro_export] +macro_rules! cmd_out { + (parse = String; err_msg = $err:expr; $($cmd:tt)*) => { + $crate::cmd_out!( + @start + $err; + |output| { + $crate::cmd_out!( + @check + output; + $err; + ::std::string::String::from_utf8(output.stdout) + .into_diagnostic() + .wrap_err("When reading to string") + .wrap_err_with(|| $err) + ) + }; + $($cmd)* + ) + }; + (parse = $out_typ:ty; err_msg = $err:expr; $($cmd:tt)*) => { + $crate::cmd_out!( + @start + $err; + |output| { + $crate::cmd_out!( + @check + output; + $err; + ::std::string::String::from_utf8(output.stdout) + .into_diagnostic() + .wrap_err("When reading to string") + .wrap_err_with(|| $err) + .and_then(|output| { + output.parse::<$out_typ>() + .into_diagnostic() + .wrap_err("When parsing") + .wrap_err_with(|| $err) + }) + ) + }; + $($cmd)* + ) + }; + (from_json = $out_typ:ty; err_msg = $err:expr; $($cmd:tt)*) => { + $crate::cmd_out!( + @start + $err; + |output| { + $crate::cmd_out!( + @check + output; + $err; + ::serde_json::from_slice::<$out_typ>(&output.stdout) + .into_diagnostic() + .wrap_err("When deserializing") + .wrap_err_with(|| $err) + ) + }; + $($cmd)* + ) + }; + (err_msg = $err:expr; $($cmd:tt)*) => { + $crate::cmd_out!( + @start + $err; + |output| { + $crate::cmd_out!( + @check + output; + $err; + Ok(()) + ) + }; + $($cmd)* + ) + }; + (@check $output:ident; $err:expr; $map:expr) => { + if !$output.status.success() { + Err(::miette::miette!( + "{}\n{}", + $err, + ::std::string::String::from_utf8_lossy(&$output.stderr) + )) + } else { + $map + } + }; + (@start $err:expr; $and_then:expr; $($cmd:tt)*) => { + { + use ::miette::{Context, IntoDiagnostic}; + { + let mut _c = ::comlexr::cmd!($($cmd)*); + ::log::trace!("{_c:#?}"); + dbg!(&_c); + _c.stderr(::std::process::Stdio::inherit()); + _c + } + .output() + .into_diagnostic() + .wrap_err("When calling the command") + .wrap_err_with(|| $err) + .and_then($and_then) + } + } +} diff --git a/utils/src/platform.rs b/utils/src/platform.rs index 06c15ae0..4d0cfc88 100644 --- a/utils/src/platform.rs +++ b/utils/src/platform.rs @@ -1,4 +1,5 @@ use std::{ + ops::Deref, path::{Path, PathBuf}, str::FromStr, }; @@ -8,6 +9,43 @@ use miette::bail; use oci_client::Reference; use serde::{Deserialize, Serialize}; +#[derive(Debug, Clone)] +pub struct PlatformList(Vec); + +impl From for PlatformList +where + PL: AsRef<[Platform]>, +{ + fn from(value: PL) -> Self { + let value = value.as_ref(); + if value.is_empty() { + Self(vec![Platform::default()]) + } else { + Self(value.to_vec()) + } + } +} + +impl From for Vec { + fn from(value: PlatformList) -> Self { + value.0 + } +} + +impl Deref for PlatformList { + type Target = [Platform]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Default for PlatformList { + fn default() -> Self { + Self(vec![Platform::default()]) + } +} + #[derive(Debug, Clone, Copy, ValueEnum, PartialEq, Eq, Hash)] pub enum Platform { #[value(name = "linux/amd64")] @@ -91,13 +129,18 @@ impl Platform { } } + #[must_use] + pub fn tag(&self) -> String { + self.to_string().replace('/', "_") + } + /// Get a tag friendly version of the platform. #[must_use] pub fn tagged_image(&self, image: &Reference) -> Reference { Reference::with_tag( image.registry().to_string(), image.repository().to_string(), - format!("{}_{self}", image.tag().unwrap_or("latest")).replace('/', "_"), + format!("{}_{}", image.tag().unwrap_or("latest"), self.tag()), ) }