diff --git a/third_party/copybara/.bazelproject b/third_party/copybara/.bazelproject new file mode 100644 index 0000000000..fda37e1bc2 --- /dev/null +++ b/third_party/copybara/.bazelproject @@ -0,0 +1,12 @@ +directories: + copybara/integration + java/com/google/copybara + javatests/com/google/copybara + third_party + +targets: + //copybara/integration/... + //java/com/google/copybara/... + //javatests/com/google/copybara/... + //third_party/... + diff --git a/third_party/copybara/.bazelrc b/third_party/copybara/.bazelrc new file mode 100644 index 0000000000..85d6e829f3 --- /dev/null +++ b/third_party/copybara/.bazelrc @@ -0,0 +1,11 @@ +# Options applied to all Bazel invocations in the workspace. +# Enforces UTF-8 encoding in bazel tests. +test --test_env='LC_ALL=en_US.UTF-8' +test --test_env='LANG=en_US.UTF-8' +test --jvmopt='-Dsun.jnu.encoding=UTF-8' +test --jvmopt='-Dfile.encoding=UTF-8' +build --test_env='LC_ALL=en_US.UTF-8' +build --jvmopt='-Dsun.jnu.encoding=UTF-8' +build --jvmopt='-Dfile.encoding=UTF-8' +build --test_env='LANG=en_US.UTF-8' +test --test_env=PATH diff --git a/third_party/copybara/.docker/entrypoint.sh b/third_party/copybara/.docker/entrypoint.sh new file mode 100644 index 0000000000..ffd2e976ec --- /dev/null +++ b/third_party/copybara/.docker/entrypoint.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +java -jar /opt/copybara/copybara_deploy.jar $COPYBARA_OPTIONS $COPYBARA_SUBCOMMAND $COPYBARA_CONFIG $COPYBARA_WORKFLOW $COPYBARA_SOURCEREF diff --git a/third_party/copybara/.gitignore b/third_party/copybara/.gitignore new file mode 100644 index 0000000000..034bc964b7 --- /dev/null +++ b/third_party/copybara/.gitignore @@ -0,0 +1,5 @@ +bazel-* +.idea +*.iml +*.pyc +.ijwb diff --git a/third_party/copybara/AUTHORS b/third_party/copybara/AUTHORS new file mode 100644 index 0000000000..f993a395f5 --- /dev/null +++ b/third_party/copybara/AUTHORS @@ -0,0 +1,9 @@ +# This the official list of Copybara authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as: +# Name or Organization +# The email address is not required for organizations. + +Google Inc. diff --git a/third_party/copybara/CONTRIBUTING.md b/third_party/copybara/CONTRIBUTING.md new file mode 100644 index 0000000000..c39eaf1d3a --- /dev/null +++ b/third_party/copybara/CONTRIBUTING.md @@ -0,0 +1,20 @@ +# Contributing guidelines + +## How to become a contributor and submit your own code + +### Contributor License Agreements + +We'd love to accept your patches! Before we can take them, we have to jump a couple of legal hurdles. + +Please fill out either the individual or corporate Contributor License Agreement (CLA). + + * If you are an individual writing original source code and you're sure you own the intellectual property, then you'll need to sign an [individual CLA](http://code.google.com/legal/individual-cla-v1.0.html). + * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](http://code.google.com/legal/corporate-cla-v1.0.html). + +Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll be able to accept your pull requests. + +***NOTE***: Only original source code from you and other people that have signed the CLA can be accepted into the main repository. + +### Contributing code + +If you have improvements to Copybara, send us your pull requests! diff --git a/third_party/copybara/Dockerfile b/third_party/copybara/Dockerfile new file mode 100644 index 0000000000..20082a93a0 --- /dev/null +++ b/third_party/copybara/Dockerfile @@ -0,0 +1,50 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM l.gcr.io/google/bazel:latest AS build + +WORKDIR /usr/src/copybara + +COPY . . + +RUN bazel build //java/com/google/copybara:copybara_deploy.jar \ + && mkdir -p /tmp/copybara \ + && cp bazel-bin/java/com/google/copybara/copybara_deploy.jar /tmp/copybara/ + +# Fails currently +# RUN bazel test //... + +FROM golang:latest AS buildtools + +RUN go get github.com/bazelbuild/buildtools/buildozer +RUN go get github.com/bazelbuild/buildtools/buildifier + +FROM openjdk:8-jre-slim +ENV COPYBARA_CONFIG=copy.bara.sky \ + COPYBARA_SUBCOMMAND=migrate \ + COPYBARA_OPTIONS='' \ + COPYBARA_WORKFLOW=default \ + COPYBARA_SOURCEREF='' +COPY --from=build /tmp/copybara/ /opt/copybara/ +COPY --from=buildtools /go/bin/buildozer /go/bin/buildifier /usr/bin/ +COPY .docker/entrypoint.sh /usr/local/bin/copybara + +RUN chmod +x /usr/local/bin/copybara + +# Install git for fun times +RUN apt-get update \ + && apt-get install -y git \ + && apt-get clean + +WORKDIR /usr/src/app diff --git a/third_party/copybara/LICENSE b/third_party/copybara/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/third_party/copybara/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/copybara/README.md b/third_party/copybara/README.md new file mode 100644 index 0000000000..1a1715121e --- /dev/null +++ b/third_party/copybara/README.md @@ -0,0 +1,165 @@ +# Copybara + +*A tool for transforming and moving code between repositories.* + +Copybara is a tool used internally at Google. It transforms and moves code between repositories. + +Often, source code needs to exist in multiple repositories, and Copybara allows you to transform +and move source code between these repositories. A common case is a project that involves +maintaining a confidential repository and a public repository in sync. + +Copybara requires you to choose one of the repositories to be the authoritative repository, so that +there is always one source of truth. However, the tool allows contributions to any repository, and +any repository can be used to cut a release. + +The most common use case involves repetitive movement of code from one repository to another. +Copybara can also be used for moving code once to a new repository. + +Examples uses of Copybara include: + + - Importing sections of code from a confidential repository to a public repository. + + - Importing code from a public repository to a confidential repository. + + - Importing a change from a non-authoritative repository into the authoritative repository. When + a change is made in the non-authoritative repository (for example, a contributor in the public + repository), Copybara transforms and moves that change into the appropriate place in the + authoritative repository. Any merge conflicts are dealt with in the same way as an out-of-date + change within the authoritative repository. + +Currently, the only supported type of repository is Git. Copybara also supports reading from Mercurial repositories, but the feature is still experimental. +Support for other repositories types will be added in the future. + +## Example + +```python +core.workflow( + name = "default", + origin = git.github_origin( + url = "https://github.com/google/copybara.git", + ref = "master", + ), + destination = git.destination( + url = "file:///tmp/foo", + ), + + # Copy everything but don't remove a README_INTERNAL.txt file if it exists. + destination_files = glob(["third_party/copybara/**"], exclude = ["README_INTERNAL.txt"]), + + authoring = authoring.pass_thru("Default email "), + transformations = [ + core.replace( + before = "//third_party/bazel/bashunit", + after = "//another/path:bashunit", + paths = glob(["**/BUILD"])), + core.move("", "third_party/copybara") + ], +) +``` + +Run: + +```shell +$ (mkdir /tmp/foo ; cd /tmp/foo ; git init --bare) +$ copybara copy.bara.sky +``` + +## Getting Started using Copybara + +Copybara doesn't have a release process yet, so you need to compile from HEAD. In order to do that +you need: + + * [Install JDK 8](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html). + * [Install Bazel](https://docs.bazel.build/versions/master/install.html). + * Clone the copybara source locally: + * `git clone https://github.com/google/copybara.git` + * Build: + * `bazel build //java/com/google/copybara`. + * `bazel build //java/com/google/copybara:copybara_deploy.jar` to create a executable uberjar. + * Tests: `bazel test //...` if you want to ensure you are not using a broken version. + +### Using Intellij with Bazel plugin + +If you use Intellij and the Bazel plugin, use this project configuration: + +``` +directories: + copybara/integration + java/com/google/copybara + javatests/com/google/copybara + third_party + +targets: + //copybara/integration/... + //java/com/google/copybara/... + //javatests/com/google/copybara/... + //third_party/... +``` + +Note that configuration files can be stored in any place, even in a local folder. We recommend to +use a VCS (like git) to store them; treat them as source code. + +### Using Docker to build and run Copybara + +*NOTE: Docker use is currently experimental, and we encourage feedback or contributions.* + +You can build copybara using Docker like so + +``` +docker build --rm -t copybara . +``` + +Once this has finished building you can run the image like so from the root of the code you are trying to use Copybara on: + +``` +docker run -it -v "$(pwd)":/usr/src/app copybara copybara + +``` + +A few environment variables exist to allow you to change how you run copybara: +* `COPYBARA_CONFIG=copy.bara.sky` + * allows you to specify a path to a config file, defaults to root `copy.bara.sky` +* `COPYBARA_SUBCOMMAND=migrate` + * allows you to change the command run, defaults to `migrate` +* `COPYBARA_OPTIONS=''` + * allows you to specify options for copybara, defaults to none +* `COPYBARA_WORKFLOW=default` + * allows you to specify the workflow to run, defaults to `default` +* `COPYBARA_SOURCEREF=''` + * allows you to specify the sourceref, defaults to none + +``` +docker run + -e COPYBARA_CONFIG='other.config.sky' + -e COPYBARA_SUBCOMMAND='validate' + -v "$(pwd)":/usr/src/app + -it copybara copybara +``` + +#### Git Config and Credentials + +There are a number of ways by which to share your git config and ssh credentials with the docker container, an example with OS X is below: + +``` +docker run + -v ~/.ssh:/root/.ssh + -v ~/.gitconfig:/root/.gitconfig + -v "$(pwd)":/usr/src/app + -it copybara copybara +``` + +## Documentation + +We are still working on the documentation. Here are some resources: + + * [Reference documentation](docs/reference.md) + * [Examples](docs/examples.md) + +## Contact us + +If you have any questions about how Copybara works please contact us at our [mailing list](https://groups.google.com/forum/#!forum/copybara-discuss) + +## Optional tips + + * If you want to see the test errors in Bazel, instead of having to cat the logs, add this line to your `~/.bazelrc: *test --test_output=streamed*`. + diff --git a/third_party/copybara/WORKSPACE b/third_party/copybara/WORKSPACE new file mode 100644 index 0000000000..4fd4964af2 --- /dev/null +++ b/third_party/copybara/WORKSPACE @@ -0,0 +1,186 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +workspace(name = "copybara") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//third_party:bazel.bzl", "bazel_sha256", "bazel_version") + +RULES_JVM_EXTERNAL_TAG = "3.0" + +RULES_JVM_EXTERNAL_SHA = "62133c125bf4109dfd9d2af64830208356ce4ef8b165a6ef15bbff7460b35c3a" + +http_archive( + name = "rules_jvm_external", + sha256 = RULES_JVM_EXTERNAL_SHA, + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, + url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, +) + +load("@rules_jvm_external//:defs.bzl", "maven_install") + +maven_install( + artifacts = [ + "com.beust:jcommander:1.48", + "com.google.auto.value:auto-value-annotations:1.6.3", + "com.google.auto.value:auto-value:1.6.3", + "com.google.auto:auto-common:0.10", + "com.google.code.findbugs:jsr305:3.0.2", + "com.google.code.gson:gson:jar:2.8.5", + "com.google.flogger:flogger-system-backend:0.3.1", + "com.google.flogger:flogger:0.3.1", + "com.google.guava:failureaccess:1.0.1", + "com.google.guava:guava-testlib:27.1-jre", + "com.google.guava:guava:27.1-jre", + "com.google.http-client:google-http-client-gson:jar:1.27.0", + "com.google.http-client:google-http-client-test:jar:1.27.0", + "com.google.http-client:google-http-client:jar:1.27.0", + "com.google.jimfs:jimfs:1.1", + "com.google.re2j:re2j:1.2", + "com.google.truth:truth:0.45", + "com.googlecode.java-diff-utils:diffutils:1.3.0", + "commons-codec:commons-codec:jar:1.11", + "junit:junit:4.13", + "net.bytebuddy:byte-buddy-agent:1.9.10", + "net.bytebuddy:byte-buddy:1.9.10", + "org.mockito:mockito-core:2.28.2", + "org.objenesis:objenesis:1.0", + ], + repositories = [ + "https://maven.google.com", + "https://repo1.maven.org/maven2", + ], +) + +# LICENSE: The Apache Software License, Version 2.0 +http_archive( + name = "io_bazel", + sha256 = bazel_sha256, + strip_prefix = "bazel-" + bazel_version, + # patch required to avoid depending on broken @io_bazel//src/main/java/com/google/devtools/build/lib/syntax:libcpu_profiler.so + patches = ["@copybara//third_party:bazel.patch"], + url = "https://github.com/bazelbuild/bazel/archive/" + bazel_version + ".zip", +) + +# LICENSE: The Apache Software License, Version 2.0 +# Buildifier and friends: +http_archive( + name = "buildtools", + sha256 = "fc9c2375fc9d50e5dd2f535b55dd25f12839a3043e7bd09a43ef7180b5670502", + strip_prefix = "buildtools-90de5e7001fbdfec29d4128bb508e01169f46950", + url = "https://github.com/bazelbuild/buildtools/archive/90de5e7001fbdfec29d4128bb508e01169f46950.zip", +) + +EXPORT_WORKSPACE_IN_BUILD_FILE = [ + "test -f BUILD && chmod u+w BUILD || true", + "echo >> BUILD", + "echo 'exports_files([\"WORKSPACE\"], visibility = [\"//visibility:public\"])' >> BUILD", +] + +EXPORT_WORKSPACE_IN_BUILD_FILE_WIN = [ + "Add-Content -Path BUILD -Value \"`nexports_files([`\"WORKSPACE`\"], visibility = [`\"//visibility:public`\"])`n\" -Force", +] + +# Stuff used by Bazel Starlark syntax package transitively: +# LICENSE: The Apache Software License, Version 2.0 +http_archive( + name = "com_google_protobuf", + patch_args = ["-p1"], + patches = ["@io_bazel//third_party/protobuf:3.11.3.patch"], + patch_cmds = EXPORT_WORKSPACE_IN_BUILD_FILE, + patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_FILE_WIN, + sha256 = "cf754718b0aa945b00550ed7962ddc167167bd922b842199eeb6505e6f344852", + strip_prefix = "protobuf-3.11.3", + urls = [ + "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v3.11.3.tar.gz", + "https://github.com/protocolbuffers/protobuf/archive/v3.11.3.tar.gz", + ], +) + +# Stuff used by Buildifier transitively: +# LICENSE: The Apache Software License, Version 2.0 +http_archive( + name = "io_bazel_rules_go", + sha256 = "b27e55d2dcc9e6020e17614ae6e0374818a3e3ce6f2024036e688ada24110444", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.21.0/rules_go-v0.21.0.tar.gz", + "https://github.com/bazelbuild/rules_go/releases/download/v0.21.0/rules_go-v0.21.0.tar.gz", + ], +) + +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") + +go_rules_dependencies() + +go_register_toolchains() + +# LICENSE: The Apache Software License, Version 2.0 +http_archive( + name = "bazel_gazelle", + sha256 = "86c6d481b3f7aedc1d60c1c211c6f76da282ae197c3b3160f54bd3a8f847896f", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz", + ], +) + +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") + +gazelle_dependencies() + +# LICENSE: The Apache Software License, Version 2.0 +go_repository( + name = "skylark_syntax", + importpath = "go.starlark.net", + sum = "h1:Qoe+9POtDT51UBQ8XEnS9QKeHDQzEl2QRh3eok9R4aw=", + version = "v0.0.0-20200203144150-6677ee5c7211", +) + +# LICENSE: The Apache Software License, Version 2.0 +http_archive( + name = "rules_pkg", + sha256 = "5bdc04987af79bd27bc5b00fe30f59a858f77ffa0bd2d8143d5b31ad8b1bd71c", + url = "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.0/rules_pkg-0.2.0.tar.gz", +) + +# LICENSE: The Apache Software License, Version 2.0 +http_archive( + name = "rules_java", + sha256 = "52423cb07384572ab60ef1132b0c7ded3a25c421036176c0273873ec82f5d2b2", + url = "https://github.com/bazelbuild/rules_java/releases/download/0.1.0/rules_java-0.1.0.tar.gz", +) + +# LICENSE: The Apache Software License, Version 2.0 +http_archive( + name = "rules_python", + sha256 = "f7402f11691d657161f871e11968a984e5b48b023321935f5a55d7e56cf4758a", + strip_prefix = "rules_python-9d68f24659e8ce8b736590ba1e4418af06ec2552", + url = "https://github.com/bazelbuild/rules_python/archive/9d68f24659e8ce8b736590ba1e4418af06ec2552.zip", +) + +# LICENSE: The Apache Software License, Version 2.0 +http_archive( + name = "rules_cc", + sha256 = "faa25a149f46077e7eca2637744f494e53a29fe3814bfe240a2ce37115f6e04d", + strip_prefix = "rules_cc-ea5c5422a6b9e79e6432de3b2b29bbd84eb41081", + url = "https://github.com/bazelbuild/rules_cc/archive/ea5c5422a6b9e79e6432de3b2b29bbd84eb41081.zip", +) + +# LICENSE: The Apache Software License, Version 2.0 +http_archive( + name = "rules_proto", + sha256 = "7d05492099a4359a6006d1b89284d34b76390c3b67d08e30840299b045838e2d", + strip_prefix = "rules_proto-9cd4f8f1ede19d81c6d48910429fe96776e567b1", + url = "https://github.com/bazelbuild/rules_proto/archive/9cd4f8f1ede19d81c6d48910429fe96776e567b1.zip", +) diff --git a/third_party/copybara/ci/ubuntu/continuous.cfg b/third_party/copybara/ci/ubuntu/continuous.cfg new file mode 100644 index 0000000000..0a5013a9bc --- /dev/null +++ b/third_party/copybara/ci/ubuntu/continuous.cfg @@ -0,0 +1,16 @@ +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Location of the continuous bash script in Git. +build_file: "copybara/ci/ubuntu/continuous.sh" diff --git a/third_party/copybara/ci/ubuntu/continuous.sh b/third_party/copybara/ci/ubuntu/continuous.sh new file mode 100755 index 0000000000..b9381fe339 --- /dev/null +++ b/third_party/copybara/ci/ubuntu/continuous.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2019 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fail on any error. +set -e +# Display commands being run. +set -x + +cd git/copybara +./cibuild.sh diff --git a/third_party/copybara/cibuild.sh b/third_party/copybara/cibuild.sh new file mode 100755 index 0000000000..3b38ffd6d1 --- /dev/null +++ b/third_party/copybara/cibuild.sh @@ -0,0 +1,44 @@ +#!/bin/bash -e + +# DISCLAIMER: This is not the Google Cloud Build script +# that you are looking for. + + +function log() { + d=$(date +'%Y-%m-%d %H:%M:%S') + echo $d" "$1 +} + +log "Running Copybara tests" + +log "Fetching dependencies" +log "Running apt-get update --fix-missing" +sudo apt-get update --fix-missing +# Mercurial does not have an up-to-date .deb package +# The official release needs to be installed with pip. +sudo apt-get -y install python-pip +sudo apt-get install locales +sudo pip install --ignore-installed --upgrade mercurial + +log "Extracting Bazel" +# Only because first time it extracts the installation +bazel version + +echo "-----------------------------------" +echo "Versions:" +hg --version | grep "(version" | sed 's/.*[(]version \([^ ]*\)[)].*/Mercurial: \1/' +git --version | sed 's/git version/Git:/' +bazel version | grep "Build label" | sed 's/Build label:/Bazel:/' +echo "-----------------------------------" + +log "Setting Locale" +export LANGUAGE=en_US.UTF-8 +export LANG=en_US.UTF-8 +export LC_ALL=en_US.UTF-8 +locale-gen en_US.UTF-8 +sudo update-locale LANG=en_US.UTF-8 + +log "Running Bazel" +bazel test ... --test_output=errors --sandbox_tmpfs_path=/tmp -j 1000 + +log "Done" diff --git a/third_party/copybara/cloudbuild.sh b/third_party/copybara/cloudbuild.sh new file mode 100755 index 0000000000..c1c8ab6410 --- /dev/null +++ b/third_party/copybara/cloudbuild.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e + +# Build script for Google Cloud Build + +function log() { + d=$(date +'%Y-%m-%d %H:%M:%S') + echo $d" "$1 +} + +log "Running Copybara tests" + +log "Fetching dependencies" +log "Running apt-get update --fix-missing" +apt-get update --fix-missing +# Mercurial does not have an up-to-date .deb package +# The official release needs to be installed with pip. +apt-get -y install python-pip +apt-get install locales +pip install mercurial + +log "Extracting Bazel" +# Only because first time it extracts the installation +bazel version + +echo "-----------------------------------" +echo "Versions:" +hg --version | grep "(version" | sed 's/.*[(]version \([^ ]*\)[)].*/Mercurial: \1/' +git --version | sed 's/git version/Git:/' +bazel version | grep "Build label" | sed 's/Build label:/Bazel:/' +echo "-----------------------------------" + +log "Setting Locale" +export LANGUAGE=en_US.UTF-8 +export LANG=en_US.UTF-8 +export LC_ALL=en_US.UTF-8 +locale-gen en_US.UTF-8 +update-locale LANG=en_US.UTF-8 + +log "Running Bazel" +bazel "$@" + +log "Done" diff --git a/third_party/copybara/cloudbuild.yaml b/third_party/copybara/cloudbuild.yaml new file mode 100644 index 0000000000..042b61567e --- /dev/null +++ b/third_party/copybara/cloudbuild.yaml @@ -0,0 +1,9 @@ +# GCB configuration file +# To learn more about GCB, go to https://cloud.google.com/container-builder/docs/ +steps: +- name: gcr.io/cloud-builders/bazel + entrypoint: "bash" + args: ["-c", "./cloudbuild.sh test ... --test_output=errors --sandbox_tmpfs_path=/tmp -j 1000"] +options: + machine_type: N1_HIGHCPU_32 +timeout: 30m diff --git a/third_party/copybara/copybara/integration/BUILD b/third_party/copybara/copybara/integration/BUILD new file mode 100644 index 0000000000..690c696994 --- /dev/null +++ b/third_party/copybara/copybara/integration/BUILD @@ -0,0 +1,38 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache 2.0 + +sh_test( + name = "reference_doc_test", + srcs = ["reference_doc_test.sh"], + data = [ + "//docs:reference.md", + "//java/com/google/copybara:docs.md", + "//third_party/bazel/bashunit", + ], + visibility = ["//visibility:public"], +) + +sh_test( + name = "tool_test", + srcs = ["tool_test.sh"], + data = [ + "//java/com/google/copybara", + "//third_party/bazel/bashunit", + ], + shard_count = 30, + tags = ["local"], + visibility = ["//visibility:public"], +) diff --git a/third_party/copybara/copybara/integration/hello.t b/third_party/copybara/copybara/integration/hello.t new file mode 100644 index 0000000000..7711acf761 --- /dev/null +++ b/third_party/copybara/copybara/integration/hello.t @@ -0,0 +1,16 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + $ echo hello + hello diff --git a/third_party/copybara/copybara/integration/reference_doc_test.sh b/third_party/copybara/copybara/integration/reference_doc_test.sh new file mode 100755 index 0000000000..8701184eba --- /dev/null +++ b/third_party/copybara/copybara/integration/reference_doc_test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source "${TEST_SRCDIR}/copybara/third_party/bazel/bashunit/unittest.bash" + +function test_reference_doc_generated() { + doc=${TEST_SRCDIR}/copybara/java/com/google/copybara/docs.md + source_doc=${TEST_SRCDIR}/copybara/docs/reference.md + + [[ -f $doc ]] || fail "Documentation not generated" + # Check that we have table of contents and some basic modules + grep "^# Table of Contents" "$doc" > /dev/null 2>&1 || fail "Table of contents not found" + grep "^## core" "$doc" > /dev/null 2>&1 || fail "core doc not found" + grep "^### core.replace" "$doc" > /dev/null 2>&1 || fail "core.replace doc not found" + grep "before.*The text before the transformation" \ + "$doc" > /dev/null 2>&1 || fail "core.replace field doc not found" + grep "^### git.origin" "$doc" > /dev/null 2>&1 || fail "git.origin doc not found" + grep "Finds links to commits in change messages" "$doc" > /dev/null 2>&1 \ + || fail "single example not found" + + diff $doc $source_doc || fail "Generate the documentation with scripts/update_docs [-a]" +} + +run_suite "Integration tests for reference documentation generation." diff --git a/third_party/copybara/copybara/integration/test-help.t b/third_party/copybara/copybara/integration/test-help.t new file mode 100644 index 0000000000..e9889e9808 --- /dev/null +++ b/third_party/copybara/copybara/integration/test-help.t @@ -0,0 +1,38 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + $ ${TEST_SRCDIR}/java/com/google/copybara/copybara help + Usage: copybara [options] CONFIG_PATH [SOURCE_REF] + Options: + --folder-dir + Local directory to put the output of the transformation + --gerrit-change-id + ChangeId to use in the generated commit message + Default: + --git-previous-ref + Previous SHA-1 reference used for the migration. + Default: + --help + Shows this help text + Default: false + --work-dir + Directory where all the transformations will be performed. By default a + temporary directory. + -v + Verbose output. + Default: false + + Example: + copybara myproject.copybara origin/master + diff --git a/third_party/copybara/copybara/integration/tool_test.sh b/third_party/copybara/copybara/integration/tool_test.sh new file mode 100755 index 0000000000..e2554365c0 --- /dev/null +++ b/third_party/copybara/copybara/integration/tool_test.sh @@ -0,0 +1,1242 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source "${TEST_SRCDIR}/copybara/third_party/bazel/bashunit/unittest.bash" + +# This should be kept in sync with ExitCode +readonly SUCCESS=0 +readonly COMMAND_LINE_ERROR=1 +readonly CONFIGURATION_ERROR=2 +readonly REPOSITORY_ERROR=3 +readonly NO_OP=4 +readonly INTERRUPTED=8 +readonly ENVIRONMENT_ERROR=30 +readonly INTERNAL_ERROR=31 + +function run_git() { + git "$@" > $TEST_log 2>&1 || fail "Error running git" +} + +# A log configuration that outputs to the console, so that we can check the log easier +log_config=$(mktemp) +cat > $log_config <<'EOF' +handlers=java.util.logging.ConsoleHandler +java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %4$-6s %2$s %5$s%6$s%n +EOF + +# Extracted as a variable so that internal tests can extend the test with different binary +copybara_binary="${copybara_binary-"${TEST_SRCDIR}/copybara/java/com/google/copybara/copybara"}" + +function copybara() { + $copybara_binary --jvm_flag=-Djava.util.logging.config.file=$log_config "$@" \ + --output-root "$repo_storage" \ + --work-dir "$workdir" > $TEST_log 2>&1 \ + && return + + res=$? + printf 'Copybara process returned error %d:\n' $res + cat $TEST_log + return $res +} + +function set_up() { + # Avoid reusing the same directory for each tests so that we don't + # share state between tests. + cd "$(mktemp -d)" || return + # set XDG_CACHE_HOME so that we have a writeable place for our caches + export XDG_CACHE_HOME="$(mktemp -d)" + # An early check to avoid confusing test failures + git version || fail "Git doesn't seem to be installed. Cannot test without git command." + + export HOME="$(mktemp -d)" + export repo_storage=$(temp_dir storage) + export workdir=$(temp_dir workdir) + git config --global user.name 'Bara Kopi' + git config --global user.email 'bara@kopi.com' +} + +function temp_dir() { + echo "$PWD/$(mktemp -d $1.XXXXXXXXXX)" +} + +function expect_in_file() { + local regex="$1" + local file="$2" + cat "$file" > $TEST_log || fail "'$file' not found" + expect_log "$regex" || fail "Cannot find '$regex' in '$file'" +} + +# Runs Copybara and check that the exit code is the expected one. +function copybara_with_exit_code() { + local expected_code=$1 + shift + copybara "$@" && status=$? || status=$? + if [ $status -ne $expected_code ]; then + fail "Unexpected exit code $status. Expected $expected_code." + fi + return 0 +} + +function check_copybara_rev_id() { + local repo="$1" + local origin_id="$2" + ( cd $repo || return + run_git log master -1 > $TEST_log + expect_log "GitOrigin-RevId: $origin_id" + ) +} + +function test_git_tracking() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + pushd $remote || return + run_git init . + echo "first version for food and foooooo" > test.txt + mkdir subdir + echo "first version for food and fools" > subdir/test.txt + run_git add test.txt subdir/test.txt + run_git commit -m "first commit" + first_commit=$(run_git rev-parse HEAD) + popd || return + + cat > copy.bara.sky <"), + transformations = [ + core.replace( + before = "food", + after = "drink" + ), + core.replace( + before = "f\${os}o", + after = "bar\${os}", + regex_groups = { + "os" : "o+" + }, + ) + ], +) +EOF + + copybara copy.bara.sky --force + + expect_log '\[ 1/2\] Transform Replace food' + expect_log '\[ 2/2\] Transform Replace f\${os}o' + + # Make sure we don't get detached head warnings polluting the log. + expect_not_log "You are in 'detached HEAD' state" + + [[ -f $workdir/checkout/test.txt ]] || fail "Checkout was not successful" + cat $workdir/checkout/test.txt > $TEST_log + expect_log "first version for drink and barooooo" + cat $workdir/checkout/subdir/test.txt > $TEST_log + expect_in_file "first version for drink and barols" $workdir/checkout/subdir/test.txt + check_copybara_rev_id "$destination" "$first_commit" + + # Do a new modification and check that we are tracking the changes to the branch + pushd $remote || return + echo "second version for food and foooooo" > test.txt + echo "second version for food and fools" > subdir/test.txt + + run_git add test.txt + run_git commit -m "second commit" + second_commit=$(git rev-parse HEAD) + popd || return + + copybara copy.bara.sky + + [[ -f $workdir/checkout/test.txt ]] || fail "Checkout was not successful" + expect_in_file "second version for drink and barooooo" $workdir/checkout/test.txt + + check_copybara_rev_id "$destination" "$second_commit" + + ( cd $destination || return + run_git show master > $TEST_log + ) + + expect_log "-first version for drink" + expect_log "+second version for drink and barooooo" +} + +function test_git_iterative() { + remote=$(temp_dir remote) + repo_storage=$(temp_dir storage) + workdir=$(temp_dir workdir) + destination=$(empty_git_bare_repo) + + pushd $remote || return + run_git init . + commit_one=$(single_file_commit "commit one" file.txt "food fooooo content1") + commit_two=$(single_file_commit "commit two" file.txt "food fooooo content2") + commit_three=$(single_file_commit "commit three" file.txt "food fooooo content3") + commit_four=$(single_file_commit "commit four" file.txt "food fooooo content4") + commit_five=$(single_file_commit "commit five" file.txt "food fooooo content5") + + popd || return + cat > copy.bara.sky <"), + mode = "ITERATIVE", +) +EOF + + copybara copy.bara.sky default $commit_three --last-rev $commit_one + + check_copybara_rev_id "$destination" "$commit_three" + + ( cd $destination || return + run_git log master~1..master > $TEST_log + ) + expect_not_log "commit two" + expect_log "commit three" + + copybara copy.bara.sky default $commit_five + + check_copybara_rev_id "$destination" "$commit_five" + + ( cd $destination || return + run_git log master~2..master~1 > $TEST_log + ) + expect_log "commit four" + + ( cd $destination || return + run_git log master~1..master > $TEST_log + ) + expect_log "commit five" + + # Running the same workflow with the same ref is no-op + copybara_with_exit_code $NO_OP copy.bara.sky default $commit_three + expect_log "No new changes to import for resolved ref" +} + +function single_file_commit() { + message=$1 + file=$2 + content=$3 + echo $content > $file + run_git add $file + run_git commit -m "$message" + git rev-parse HEAD +} + +function test_get_git_changes() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + pushd $remote || return + run_git init . + commit_one=$(single_file_commit "commit one" file.txt "food fooooo content1") + commit_two=$(single_file_commit "commit two" file.txt "food fooooo content2") + commit_three=$(single_file_commit "commit three" file.txt "food fooooo content3") + commit_four=$(single_file_commit "commit four" file.txt "food fooooo content4") + commit_five=$(single_file_commit "commit five" file.txt "food fooooo content5") + popd || return + + cat > copy.bara.sky <"), + transformations = [ + metadata.squash_notes() + ], +) +EOF + + # Before running the tool for the first time, the last imported ref is empty + copybara info copy.bara.sky default + expect_log "'workflow_default': last_migrated None - last_available $commit_five" + expect_log "Available changes (5):" + expect_log ".*${commit_one:0:10}.*commit one.*Bara Kopi .*" + expect_log ".*${commit_two:0:10}.*commit two.*Bara Kopi .*" + expect_log ".*${commit_three:0:10}.*commit three.*Bara Kopi .*" + expect_log ".*${commit_four:0:10}.*commit four.*Bara Kopi .*" + expect_log ".*${commit_five:0:10}.*commit five.*Bara Kopi .*" + + copybara copy.bara.sky default $commit_one --force + + check_copybara_rev_id "$destination" "$commit_one" + + copybara info copy.bara.sky default + expect_log "'workflow_default': last_migrated $commit_one - last_available $commit_five" + expect_log "Available changes (4):" + expect_log ".*${commit_two:0:10}.*commit two.*Bara Kopi .*" + expect_log ".*${commit_three:0:10}.*commit three.*Bara Kopi .*" + expect_log ".*${commit_four:0:10}.*commit four.*Bara Kopi .*" + expect_log ".*${commit_five:0:10}.*commit five.*Bara Kopi .*" + + ( cd $destination || return + run_git log master~1..master > $TEST_log + ) + # By default we include the whole history if last_rev cannot be found. --squash-without-history + # can be used for disabling this. + expect_log "commit one" + + copybara copy.bara.sky default $commit_four + + copybara info copy.bara.sky default + expect_log "'workflow_default': last_migrated $commit_four - last_available $commit_five" + expect_log "Available changes (1):" + expect_log ".*${commit_five:0:10}.*commit five.*Bara Kopi .*" + + check_copybara_rev_id "$destination" "$commit_four" + + ( cd $destination || return + run_git log master~1..master > $TEST_log + ) + # "commit one" should not be included because it was migrated before + expect_not_log "commit one" + expect_log "$commit_two.*commit two" + expect_log "$commit_three.*commit three" + expect_log "$commit_four.*commit four" + expect_not_log "commit five" + + copybara copy.bara.sky default $commit_five --last-rev $commit_three + + check_copybara_rev_id "$destination" "$commit_five" + + ( cd $destination || return + run_git log master~1..master > $TEST_log + ) + # We are forcing to use commit_three as the last migration. This has + # no effect in squash workflow but it changes the release notes. + expect_not_log "commit one" + expect_not_log "commit two" + expect_not_log "commit three" + expect_log "$commit_four.*commit four" + expect_log "$commit_five.*commit five" +} + +function test_can_skip_excluded_commit() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + pushd $remote || return + run_git init . + commit_master=$(single_file_commit "last rev commit" file2.txt "origin") + commit_one=$(single_file_commit "commit one" file.txt "foooo") + # Because we exclude file2.txt this is effectively an empty commit in the destination + commit_two=$(single_file_commit "commit two" file2.txt "bar") + commit_three=$(single_file_commit "commit three" file.txt "baaaz") + popd || return + + cat > copy.bara.sky <"), + mode = "ITERATIVE", +) +EOF + + copybara copy.bara.sky default $commit_three --last-rev $commit_master --force + + check_copybara_rev_id "$destination" "$commit_three" + + ( cd $destination || return + run_git log > $TEST_log + ) + expect_log "commit one" + expect_not_log "commit two" + expect_log "commit three" +} + +function empty_git_bare_repo() { + repo=$(temp_dir repo) + cd $repo || return + run_git init . --bare > $TEST_log 2>&1 || fail "Cannot create repo" + run_git --work-tree="$(mktemp -d)" commit --allow-empty -m "Empty repo" \ + > $TEST_log 2>&1 || fail "Cannot commit to empty repo" + echo $repo +} + +function prepare_glob_tree() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + ( cd $remote || return + run_git init . + echo "foo" > test.txt + echo "foo" > test.java + mkdir -p folder/subfolder + echo "foo" > folder/test.txt + echo "foo" > folder/test.java + echo "foo" > folder/subfolder/test.txt + echo "foo" > folder/subfolder/test.java + run_git add -A + run_git commit -m "first commit" + ) +} + +function test_regex_with_path() { + prepare_glob_tree + + cat > copy.bara.sky <"), + transformations = [ + core.replace( + before = "foo", + after = "bar", + paths = glob(['**.java']), + ) + ], +) +EOF + copybara copy.bara.sky --force + ( cd "$(mktemp -d)" || return + run_git clone $destination . + expect_in_file "foo" test.txt + expect_in_file "bar" test.java + expect_in_file "foo" folder/test.txt + expect_in_file "bar" folder/test.java + expect_in_file "foo" folder/subfolder/test.txt + expect_in_file "bar" folder/subfolder/test.java + ) +} + +function git_pull_request() { + sot=$(empty_git_bare_repo) + public=$(empty_git_bare_repo) + + cat > copy.bara.sky <"), +) + +core.workflow( + name = "import_change", + origin = git.origin( + url = "file://$public", + ref = "master", + ), + destination = git.destination( + url = "file://$sot", + fetch = "master", + push = "master", + ), + authoring = authoring.pass_thru("Copybara Team "), + mode = "CHANGE_REQUEST", +) +EOF + + # Create the SoT repo + pushd "$(mktemp -d)" || return + run_git clone $sot . + commit_one=$(single_file_commit "commit one" one.txt "content") + commit_two=$(single_file_commit "commit two" two.txt "content") + commit_three=$(single_file_commit "commit three" three.txt "content") + run_git push + popd || return + + copybara copy.bara.sky export $commit_two + + # Check that we have exported correctly the tree state as in commit_two + pushd "$(mktemp -d)" || return + run_git clone $public . + [[ -f one.txt ]] || fail "one.txt should exist in commit two" + [[ -f two.txt ]] || fail "two.txt should exist in commit two" + [[ ! -f three.txt ]] || fail "three.txt should NOT exist in commit two" + + # Create a new change on top of the public version (commit_two) + pr_request=$(single_file_commit "pull request" pr.txt "content") + run_git push + popd || return + + copybara copy.bara.sky import_change $pr_request + + # Check that the SoT contains the change from pr_request but that it has not reverted + # commit_three (SoT was ahead of public). + pushd "$(mktemp -d)" || return + run_git clone $sot . + [[ -f one.txt ]] || fail "one.txt should exist after pr import" + [[ -f two.txt ]] || fail "two.txt should exist after pr import" + [[ -f three.txt ]] || fail "three.txt should exist after pr import" + [[ -f pr.txt ]] || fail "pr.txt should exist after pr import" + popd || return +} + +function test_git_delete() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + destination=$(empty_git_bare_repo) + + ( cd $remote || return + run_git init . + echo "first version for food and foooooo" > test.txt + mkdir subdir + echo "first version" > subdir/test.txt + mkdir subdir2 + echo "first version" > subdir2/test.java + echo "first version" > subdir2/test.txt + run_git add -A + run_git commit -m "first commit" + ) + + cat > copy.bara.sky <"), + origin_files = glob(include = ["**"], exclude = ['**/*.java', 'subdir/**']), +) +EOF + copybara copy.bara.sky --force + + ( cd "$(mktemp -d)" || return + run_git clone $destination . + [[ ! -f subdir/test.txt ]] || fail "/subdir/test.txt should be deleted" + [[ ! -f subdir2/test.java ]] || fail "/subdir2/test.java should be deleted" + + [[ -f test.txt ]] || fail "/test.txt should not be deleted" + [[ ! -d subdir ]] || fail "/subdir should be deleted" + [[ -d subdir2 ]] || fail "/subdir2 should not be deleted" + [[ -f subdir2/test.txt ]] || fail "/subdir2/test.txt should not be deleted" + ) +} + +function test_reverse_sequence() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + ( cd $remote || return + run_git init . + echo "foobaz" > test.txt + run_git add -A + run_git commit -m "first commit" + ) + + cat > copy.bara.sky <"), + transformations = forward_transforms, +) + +core.workflow( + name = "reverse", + origin = git.origin( + url = "file://$destination", + ref = "master", + ), + destination = git.destination( + url = "file://$remote", + fetch = "reverse", + push = "reverse", + ), + authoring = authoring.pass_thru("Copybara Team "), + transformations = core.reverse(forward_transforms), +) +EOF + copybara copy.bara.sky forward --force + + ( cd "$(mktemp -d)" || return + run_git clone $destination . + [[ -f test.txt ]] || fail "/test.txt should exit" + expect_in_file "barbee" test.txt + ) + copybara copy.bara.sky reverse --force + + ( cd "$(mktemp -d)" || return + run_git clone $remote . + run_git checkout reverse + [[ -f test.txt ]] || fail "/test.txt should exit" + expect_in_file "foobaz" test.txt + ) +} + +function test_local_dir_destination() { + remote=$(temp_dir remote) + + ( cd $remote || return + run_git init . + echo "first version for food and foooooo" > test.txt + echo "first version" > test.txt + run_git add test.txt + run_git commit -m "first commit" + ) + mkdir destination + + cat > destination/copy.bara.sky <"), +) +EOF + + touch destination/keepme.keep + mkdir -p destination/folder + touch destination/folder/keepme.keep + touch destination/dontkeep.txt + + copybara destination/copy.bara.sky --folder-dir destination + + [[ -f destination/test.txt ]] || fail "test.txt should exist" + [[ -f destination/copy.bara.sky ]] || fail "copy.bara.sky should exist" + [[ -f destination/keepme.keep ]] || fail "keepme.keep should exist" + [[ -f destination/folder/keepme.keep ]] || fail "folder/keepme.keep should exist" + [[ ! -f destination/dontkeep.txt ]] || fail "dontkeep.txt should be deleted" +} + +function test_choose_non_default_workflow() { + remote=$(temp_dir remote) + + ( cd $remote || return + run_git init . + echo "foo" > test.txt + run_git add test.txt + run_git commit -m "first commit" + ) + mkdir destination + + cat > destination/copy.bara.sky <"), +) + +core.workflow( + name = "choochoochoose_me", + origin = git.origin( + url = "file://$remote", + ref = "master", + ), + destination = folder.destination(), + authoring = authoring.pass_thru("Copybara Team "), + transformations = [core.replace("foo", "bar")] +) +EOF + + copybara destination/copy.bara.sky choochoochoose_me --folder-dir destination + expect_in_file "bar" destination/test.txt +} + +function test_file_move() { + remote=$(temp_dir remote) + + ( cd $remote || return + run_git init . + echo "foo" > test1.txt + echo "foo" > test2.txt + run_git add test1.txt test2.txt + run_git commit -m "first commit" + ) + mkdir destination + + cat > destination/copy.bara.sky <"), + transformations = [ + core.move('test1.txt', 'test1.moved'), + core.move('test2.txt', 'test2.moved'), + ], +) +EOF + + copybara destination/copy.bara.sky --folder-dir destination + + expect_in_file "foo" destination/test1.moved + expect_in_file "foo" destination/test2.moved + [[ ! -f destination/test1.txt ]] || fail "test1.txt should have been moved" + [[ ! -f destination/test2.txt ]] || fail "test2.txt should have been moved" +} + +function test_profile() { + remote=$(temp_dir remote) + + ( cd $remote || return + run_git init . + echo "foo" > test.txt + run_git add test.txt + run_git commit -m "first commit" + ) + mkdir destination + + cat > destination/copy.bara.sky <"), + transformations = [ + core.move('test.txt', 'test.moved'), + core.move('test.moved', 'test.moved2'), + ], +) +EOF + + copybara destination/copy.bara.sky --folder-dir destination + expect_log "ioRepoTask.*PROFILE:.*[0-9]* //copybara/clean_outputdir" + expect_log "repoTask.*PROFILE:.*[0-9]* //copybara/run/default/origin.resolve_source_ref" + expect_log "doMigrate.*PROFILE:.*[0-9]* //copybara/run/default/squash/prepare_workdir" + expect_log "checkout.*PROFILE:.*[0-9]* //copybara/run/default/squash/origin.checkout" + expect_log "transform.*PROFILE:.*[0-9]* //copybara/run/default/squash/transforms/Moving test.txt" + expect_log "transform.*PROFILE:.*[0-9]* //copybara/run/default/squash/transforms/Moving test.moved" + expect_log "doMigrate.*PROFILE:.*[0-9]* //copybara/run/default/squash/transforms" + expect_log "doMigrate.*PROFILE:.*[0-9]* //copybara/run/default/squash/destination.write" + expect_log "run.*PROFILE:.*[0-9]* //copybara/run/default/squash" + expect_log "run.*PROFILE:.*[0-9]* //copybara/run" + expect_log "shutdown.*PROFILE:.*[0-9]* //copybara" + expect_in_file "foo" destination/test.moved2 +} + +function test_invalid_transformations_in_config() { + cat > copy.bara.sky <"), + transformations = [42], +) +EOF + copybara_with_exit_code $CONFIGURATION_ERROR copy.bara.sky default + expect_log "for 'transformations' element, got int, want function or transformation" +} + +function test_command_help_flag() { + copybara help + expect_log 'Usage: copybara \[options\]' + expect_log 'Example:' +} + +# We want to log the command line arguments so that it is easy to reproduce +# user issues. +function test_command_args_logged() { + copybara foo bar baz --option && fail "should fail" + expect_log 'Running: .*foo bar baz --option' +} + +function test_command_copybara_filename_no_correct_name() { + copybara_with_exit_code $CONFIGURATION_ERROR migrate somename.bzl + expect_log "Copybara config file filename should be 'copy.bara.sky'" +} + +function setup_reversible_check_workflow() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + pushd $remote || return + run_git init . + echo "Is the reverse of the reverse forward?" > test.txt + run_git add test.txt + run_git commit -m "first commit" + first_commit=$(run_git rev-parse HEAD) + popd || return + + cat > copy.bara.sky <"), + reversible_check = True, + transformations = [ + core.replace( + before = "reverse", + after = "forward" + ) + ], +) +EOF +} + +function test_reversible_check() { + setup_reversible_check_workflow + copybara_with_exit_code $CONFIGURATION_ERROR copy.bara.sky --force + expect_log "ERROR: Workflow 'default' is not reversible" +} + +function test_disable_reversible_check() { + setup_reversible_check_workflow + copybara --disable-reversible-check copy.bara.sky --force +} + +function test_config_not_found() { + copybara_with_exit_code $COMMAND_LINE_ERROR copy.bara.sky origin/master + expect_log "Configuration file not found: copy.bara.sky" +} + +#Verify that we instantiate LogConsole when System.console() is null +function test_no_ansi_console() { + copybara_with_exit_code $COMMAND_LINE_ERROR copy.bara.sky + expect_log "^[0-9]\{4\} [0-2][0-9]:[0-5][0-9]:[0-5][0-9].*" +} + +# Verify that Copybara fails if we try to read the input from the user from a writeOnly LogConsole +function test_log_console_is_write_only() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + ( cd $remote || return + run_git init . + echo "first version for food and foooooo" > test.txt + run_git add -A + run_git commit -m "first commit" + ) + + cat > copy.bara.sky <"), + ask_for_confirmation = True, +) +EOF + copybara_with_exit_code $INTERNAL_ERROR copy.bara.sky --force + expect_log "LogConsole cannot read user input if system console is not present" +} + +function run_multifile() { + config_folder=$1 + shift + flags="$*" + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + pushd $remote || return + run_git init . + echo "first version for food and foooooo" > test.txt + mkdir subdir + echo "first version" > subdir/test.txt + run_git add test.txt subdir/test.txt + run_git commit -m "first commit" + first_commit=$(run_git rev-parse HEAD) + popd || return + mkdir -p $config_folder/foo/bar/baz + mkdir -p $config_folder/baz + # Just in case: + cat > $config_folder/baz/origin.bara.sky < $config_folder/foo/remote.bara.sky < $config_folder/foo/bar/baz/origin.bara.sky < $config_folder/foo/bar/copy.bara.sky <"), +) +EOF + cd $config_folder || return + copybara foo/bar/copy.bara.sky $flags --force + + [[ -f $workdir/checkout/test.txt ]] || fail "Checkout was not successful" +} + +# Test that we can find the root when config is in a git repo +function test_multifile_git_root() { + config_folder=$(temp_dir config) + pushd $config_folder || return + run_git init . + popd || return + run_multifile $config_folder +} + +# Test that on non-git repos we can pass a flag to set the root +function test_multifile_root_cfg_flag() { + config_folder=$(temp_dir config) + run_multifile $config_folder --config-root $config_folder +} + +# Regression test: config roots that are symlinks work +function test_multifile_root_cfg_flag_symlink() { + config_folder=$(temp_dir config) + mkdir "${config_folder}/test" + ln -s "${config_folder}/test" "${config_folder}/link" + run_multifile "${config_folder}/link" --config-root "${config_folder}/link" +} + +function test_verify_match() { + prepare_glob_tree + + cat > copy.bara.sky <"), + transformations = [ + core.verify_match( + regex = "bar", + paths = glob(['**.java']), + verify_no_match = True + ) + ], +) +EOF + copybara copy.bara.sky --force +} + +function test_subcommand_parsing_fails() { + copybara_with_exit_code $COMMAND_LINE_ERROR migrate.sky copy.bara.sky + + expect_log "Invalid subcommand 'migrate.sky'" +} + +function test_command_copybara_wrong_subcommand() { + copybara_with_exit_code COMMAND_LINE_ERROR foooo + expect_log "Invalid subcommand 'foooo'. Available commands: " + expect_log "Try 'copybara help'" +} + +function test_default_command_too_few_args() { + copybara_with_exit_code $COMMAND_LINE_ERROR + expect_log "Configuration file missing for 'migrate' subcommand." + expect_log "Try 'copybara help'" +} + +function test_migrate_missing_config() { + copybara_with_exit_code $COMMAND_LINE_ERROR migrate + + expect_log "Configuration file missing for 'migrate' subcommand." + expect_log "Try 'copybara help'" +} + +function test_info_missing_config() { + copybara_with_exit_code $COMMAND_LINE_ERROR info + + expect_log "Configuration file missing for 'info' subcommand." +} + +function test_info_too_many_arguments() { + copybara_with_exit_code $COMMAND_LINE_ERROR info copy.bara.sky default foo + + expect_log "Too many arguments for subcommand 'info'" +} + +function test_validate_missing_config() { + copybara_with_exit_code $COMMAND_LINE_ERROR validate + + expect_log "Configuration file missing for 'validate' subcommand." +} + +function test_validate_too_many_arguments() { + copybara_with_exit_code $COMMAND_LINE_ERROR validate copy.bara.sky default foo + + expect_log "Too many arguments for subcommand 'validate'" +} + +function test_validate_valid() { + cat > copy.bara.sky <"), + mode = "ITERATIVE", +) +EOF + + copybara validate copy.bara.sky + + expect_log "Configuration '.*copy.bara.sky' is valid." +} + +function test_validate_invalid() { + cat > copy.bara.sky < copy.bara.sky < test.java + echo "patched" > folder/test.java + git --no-pager diff > $workdir/../diff1.patch + run_git reset --hard + expect_in_file "foo" test.java + expect_in_file "foo" folder/test.java + echo "patched again" > folder/subfolder/test.java + git --no-pager diff > $workdir/../diff2.patch + run_git reset --hard + expect_in_file "foo" folder/subfolder/test.java + ) + cat > copy.bara.sky <"), + transformations = [ + patch.apply( + patches = ["diff1.patch", "diff2.patch"], + ) + ], +) +EOF + copybara copy.bara.sky --force + ( cd "$(mktemp -d)" || return + run_git clone $destination . + expect_in_file "patched" test.java + expect_in_file "patched" folder/test.java + expect_in_file "patched again" folder/subfolder/test.java + ) +} + +function test_description_migrator() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + pushd $remote || return + run_git init . + commit_initial=$(single_file_commit "initial rev commit" file2.txt "initial") + commit_master=$(single_file_commit "last rev commit" file23.txt "origin") + commit_one=$(single_file_commit "c1 foooo origin/${commit_master} bar" file.txt "one") + + popd || return + + cat > copy.bara.sky <"), + transformations = [ + metadata.map_references( + before = "origin/\${reference}", + after = "destination/\${reference}", + regex_groups = { + "before_ref": "[0-9a-f]+", + "after_ref": "[0-9a-f]+", + } + ) + ], + mode = "ITERATIVE", +) +EOF + + copybara copy.bara.sky default $commit_master --last-rev $commit_initial + copybara copy.bara.sky default $commit_one --last-rev $commit_master + + check_copybara_rev_id "$destination" "$commit_one" + + ( cd $destination || return + run_git log > $TEST_log + ) + expect_log "c1 foooo destination/[a-f0-9]\{1,\} bar" +} + +function test_invalid_last_rev() { + remote=$(temp_dir remote) + destination=$(empty_git_bare_repo) + + pushd $remote || return + run_git init . + commit_initial=$(single_file_commit "initial rev commit" file2.txt "initial") + popd || return + + cat > copy.bara.sky <"), + mode = "ITERATIVE", +) +EOF + + copybara_with_exit_code $CONFIGURATION_ERROR copy.bara.sky default --last-rev --some-other-flag + + expect_log "Invalid refspec: --some-other-flag" +} + +run_suite "Integration tests for Copybara code sharing tool." diff --git a/third_party/copybara/docs/BUILD b/third_party/copybara/docs/BUILD new file mode 100644 index 0000000000..7cd71ad24f --- /dev/null +++ b/third_party/copybara/docs/BUILD @@ -0,0 +1,19 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:public"]) + +exports_files(["reference.md"]) diff --git a/third_party/copybara/docs/examples.md b/third_party/copybara/docs/examples.md new file mode 100644 index 0000000000..cf83ea10d7 --- /dev/null +++ b/third_party/copybara/docs/examples.md @@ -0,0 +1,237 @@ +## Contents + - [Git-to-Git import](#basic-git-to-git-import) + - [Github SSH import](#github-ssh-basic-import) + - [Mercurial to Git import](#mercurial-to-git-import) + - [Transformations](#transformations) + - [Subcommands](#subcommands) + +## Basic git-to-git import + +This example will import Copybara source code to an internal git repository +under ``$GIT/third_party/copybara``. + +Assuming you have an existing git repository. For the example in ``/tmp/foo``. But it could be +a remote one: + +```bash +mkdir /tmp/foo +cd /tmp/foo +git init --bare . +``` + +Create a ``copy.bara.sky`` config file like: + +```python +url = "https://github.com/google/copybara.git" + +core.workflow( + name = "default", + origin = git.origin( + url = url, + ref = "master", + ), + destination = git.destination( + url = "file:///tmp/foo", + fetch = "master", + push = "master", + ), + # Copy everything but don't remove a README_INTERNAL.txt file if it exists. + destination_files = glob(["third_party/copybara/**"], exclude = ["README_INTERNAL.txt"]), + + authoring = authoring.pass_thru("Default email "), + transformations = [ + core.move("", "third_party/copybara"), + ], +) +``` + +Invoke the tool like: + +```bash +copybara copy.bara.sky --force +``` + +``--force`` should only be needed for empty destination repositories or non-existent +branches in the destination. After the first import, it should be always invoked as: + +``` +copybara copy.bara.sky +``` + +## GitHub SSH basic import + +This example will import private source code to an external GitHub repository, and uses SSH. + +PROTIP: You will need to have an ssh key setup without a password to accomplish this, Copybara doesn't +currently support ssh with a password. + +Create a ``copy.bara.sky`` config file like: + +```python +# Update these references to your orginzations repos +sourceUrl = "git@github.com:organization/internal-repo.git" +destinationUrl = "git@github.com:organization/external-repo.git" + +core.workflow( + name = "default", + origin = git.origin( + url = sourceUrl, + ref = "master", + ), + destination = git.destination( + url = destinationUrl, + fetch = "master", + push = "master", + ), + # Change path to the folder you want to publish publicly + origin_files = glob(["path/to/folder/you/want/exported/**"]), + + authoring = authoring.pass_thru("Default email "), + + # Change the path here to the folder you want to publish publicly + transformations = [ + core.move("path/to/folder/you/want/exported", ""), + ], +) +``` + +Invoke the tool like: + +```bash +copybara copy.bara.sky --force +``` + +``--force`` should only be needed for empty destination repositories or non-existent +branches in the destination. After the first import, it should be always invoked as: + +``` +copybara copy.bara.sky +``` + +After running through this example, you should see all the source from +the folder you selected in the external-repo at the root. This can be helpful if you +are only trying to move a subdirectory in your git repo out for public use. + +## Mercurial to Git import +Let's set up a simple migration from a Mercurial repository to a git repository. Note that Mercurial +support is still experimental. + +In this example, we will import source code from the +[Mercurial source repository](https://www.mercurial-scm.org/repo/hg/) to a local git repository. +We'll get started by setting up a local bare git repository. + +``` +$ mkdir /tmp/gitdest +$ cd /tmp/gitdest +$ git init --bare . +``` +Next up is creating and editing a `copy.bara.sky` config file. The config file will contain the +details of our workflow. Using your text editor of choice, create and edit the config file: +``` +$ vim /tmp/copy.bara.sky +``` +We'll define in the config to pull changes from the default branch in the origin repository. +``` +core.workflow( + name = "default", + origin = hg.origin( + url = "https://www.mercurial-scm.org/repo/hg", + ref = "default", + ), + destination = git.destination( + url = "file:///tmp/gitdest", + ), + # Files that you want to import + origin_files = glob(['**']), + # Files that you want to copy + destination_files = glob(['**']), + # Set up a default author + authoring = authoring.pass_thru("Default email "), + # Import mode + mode = "SQUASH", +) +``` +Now we can run Copybara with this config to import the changes. However, since the Mercurial +repository has many commits, we can just pull default branch revisions from the most recent 15 +revisions in the repository, using the `--last-rev` flag. + +``` +$ copybara /tmp/copy.bara.sky --force --last-rev -15 +``` +If we wanted to pull all revisions from the default branch, we would omit the `--last-rev` flag. +Since we are using `SQUASH` mode, all commits from the origin repository will be "squashed" into a +single commit. + +If we navigate to our git destination repository, we can run `git log` and see the commit that +was created. +``` +$ cd /tmp/gitdest +$ git log +``` + + +## Transformations + +Let's say that we realized that we need to do some code transformations to the imported code. +We could use core.replace to do it. Here we look for ``//third_party/bazel/bashunit`` text +and we replace it with the correct destination one just for BUILD files: + + +```python +url = "https://github.com/google/copybara.git" + +core.workflow( + name = "default", + origin = git.origin( + url = url, + ref = "master", + ), + destination = git.destination( + url = "file:///tmp/foo", + fetch = "master", + push = "master", + ), + + # Copy everything but don't remove a README_INTERNAL.txt file if it exists. + destination_files = glob(["third_party/copybara/**"], exclude = ["README_INTERNAL.txt"]), + + authoring = authoring.pass_thru("Default email "), + + transformations = [ + core.replace( + before = "//third_party/bazel/bashunit", + after = "//another/path:bashunit", + paths = glob(["**/BUILD"]), + ), + core.move("", "third_party/copybara"), + ], +) +``` + +## Subcommands + +The tool accepts different subcommands, _à la_ Bazel. If no +command is specified, *migrate* is executed by default. These two commands are +equivalent: + +```shell +$ copybara copy.bara.sky +$ copybara migrate copy.bara.sky +``` + +You can validate your configuration running: + +```shell +$ copybara validate copy.bara.sky +Copybara source mover +INFO: Configuration validated. +``` + +And you can get information about a migration workflow by running: + +```shell +$ copybara info copy.bara.sky +Copybara source mover +... +INFO: Workflow 'default': last_migrated_ref 4dd20b2... +``` diff --git a/third_party/copybara/docs/reference.md b/third_party/copybara/docs/reference.md new file mode 100755 index 0000000000..160cbbf22f --- /dev/null +++ b/third_party/copybara/docs/reference.md @@ -0,0 +1,3853 @@ +# Table of Contents + + + - [author](#author) + - [authoring](#authoring) + - [authoring.overwrite](#authoring.overwrite) + - [authoring.pass_thru](#authoring.pass_thru) + - [authoring.whitelisted](#authoring.whitelisted) + - [authoring_class](#authoring_class) + - [buildozer](#buildozer) + - [buildozer.cmd](#buildozer.cmd) + - [buildozer.create](#buildozer.create) + - [buildozer.delete](#buildozer.delete) + - [buildozer.modify](#buildozer.modify) + - [change](#change) + - [ChangeMessage](#changemessage) + - [message.label_values](#message.label_values) + - [Changes](#changes) + - [Console](#console) + - [console.error](#console.error) + - [console.info](#console.info) + - [console.progress](#console.progress) + - [console.verbose](#console.verbose) + - [console.warn](#console.warn) + - [core](#core) + - [core.copy](#core.copy) + - [core.dynamic_feedback](#core.dynamic_feedback) + - [core.dynamic_transform](#core.dynamic_transform) + - [core.fail_with_noop](#core.fail_with_noop) + - [core.feedback](#core.feedback) + - [core.filter_replace](#core.filter_replace) + - [core.format](#core.format) + - [core.move](#core.move) + - [core.remove](#core.remove) + - [core.replace](#core.replace) + - [core.replace_mapper](#core.replace_mapper) + - [core.reverse](#core.reverse) + - [core.todo_replace](#core.todo_replace) + - [core.transform](#core.transform) + - [core.verify_match](#core.verify_match) + - [core.workflow](#core.workflow) + - [destination_effect](#destination_effect) + - [destination_reader](#destination_reader) + - [destination_reader.copy_destination_files](#destination_reader.copy_destination_files) + - [destination_reader.read_file](#destination_reader.read_file) + - [destination_ref](#destination_ref) + - [endpoint](#endpoint) + - [endpoint.new_destination_ref](#endpoint.new_destination_ref) + - [endpoint.new_origin_ref](#endpoint.new_origin_ref) + - [endpoint_provider](#endpoint_provider) + - [feedback.action_result](#feedback.action_result) + - [feedback.context](#feedback.context) + - [feedback.context.error](#feedback.context.error) + - [feedback.context.noop](#feedback.context.noop) + - [feedback.context.record_effect](#feedback.context.record_effect) + - [feedback.context.success](#feedback.context.success) + - [feedback.finish_hook_context](#feedback.finish_hook_context) + - [feedback.finish_hook_context.record_effect](#feedback.finish_hook_context.record_effect) + - [feedback.revision_context](#feedback.revision_context) + - [filter_replace](#filter_replace) + - [folder](#folder) + - [folder.destination](#folder.destination) + - [folder.origin](#folder.origin) + - [format](#format) + - [format.buildifier](#format.buildifier) + - [gerritapi.AccountInfo](#gerritapi.accountinfo) + - [gerritapi.ApprovalInfo](#gerritapi.approvalinfo) + - [gerritapi.ChangeInfo](#gerritapi.changeinfo) + - [gerritapi.ChangeMessageInfo](#gerritapi.changemessageinfo) + - [gerritapi.ChangesQuery](#gerritapi.changesquery) + - [gerritapi.CommitInfo](#gerritapi.commitinfo) + - [gerritapi.GitPersonInfo](#gerritapi.gitpersoninfo) + - [gerritapi.LabelInfo](#gerritapi.labelinfo) + - [gerritapi.ParentCommitInfo](#gerritapi.parentcommitinfo) + - [gerritapi.ReviewResult](#gerritapi.reviewresult) + - [gerritapi.RevisionInfo](#gerritapi.revisioninfo) + - [gerrit_api_obj](#gerrit_api_obj) + - [gerrit_api_obj.get_change](#gerrit_api_obj.get_change) + - [gerrit_api_obj.list_changes_by_commit](#gerrit_api_obj.list_changes_by_commit) + - [gerrit_api_obj.post_review](#gerrit_api_obj.post_review) + - [git](#git) + - [git.destination](#git.destination) + - [git.gerrit_api](#git.gerrit_api) + - [git.gerrit_destination](#git.gerrit_destination) + - [git.gerrit_origin](#git.gerrit_origin) + - [git.gerrit_trigger](#git.gerrit_trigger) + - [git.github_api](#git.github_api) + - [git.github_destination](#git.github_destination) + - [git.github_origin](#git.github_origin) + - [git.github_pr_destination](#git.github_pr_destination) + - [git.github_pr_origin](#git.github_pr_origin) + - [git.github_trigger](#git.github_trigger) + - [git.integrate](#git.integrate) + - [git.latest_version](#git.latest_version) + - [git.mirror](#git.mirror) + - [git.origin](#git.origin) + - [git.review_input](#git.review_input) + - [github_api_obj](#github_api_obj) + - [github_api_obj.add_label](#github_api_obj.add_label) + - [github_api_obj.create_status](#github_api_obj.create_status) + - [github_api_obj.delete_reference](#github_api_obj.delete_reference) + - [github_api_obj.get_authenticated_user](#github_api_obj.get_authenticated_user) + - [github_api_obj.get_check_runs](#github_api_obj.get_check_runs) + - [github_api_obj.get_combined_status](#github_api_obj.get_combined_status) + - [github_api_obj.get_commit](#github_api_obj.get_commit) + - [github_api_obj.get_pull_request_comment](#github_api_obj.get_pull_request_comment) + - [github_api_obj.get_pull_request_comments](#github_api_obj.get_pull_request_comments) + - [github_api_obj.get_pull_requests](#github_api_obj.get_pull_requests) + - [github_api_obj.get_reference](#github_api_obj.get_reference) + - [github_api_obj.get_references](#github_api_obj.get_references) + - [github_api_obj.update_pull_request](#github_api_obj.update_pull_request) + - [github_api_obj.update_reference](#github_api_obj.update_reference) + - [Globals](#globals) + - [glob](#glob) + - [new_author](#new_author) + - [parse_message](#parse_message) + - [hg](#hg) + - [hg.origin](#hg.origin) + - [mapping_function](#mapping_function) + - [metadata](#metadata) + - [metadata.add_header](#metadata.add_header) + - [metadata.expose_label](#metadata.expose_label) + - [metadata.map_author](#metadata.map_author) + - [metadata.map_references](#metadata.map_references) + - [metadata.remove_label](#metadata.remove_label) + - [metadata.replace_message](#metadata.replace_message) + - [metadata.restore_author](#metadata.restore_author) + - [metadata.save_author](#metadata.save_author) + - [metadata.scrubber](#metadata.scrubber) + - [metadata.squash_notes](#metadata.squash_notes) + - [metadata.use_last_change](#metadata.use_last_change) + - [metadata.verify_match](#metadata.verify_match) + - [origin_ref](#origin_ref) + - [patch](#patch) + - [patch.apply](#patch.apply) + - [Path](#path) + - [path.read_symlink](#path.read_symlink) + - [path.relativize](#path.relativize) + - [path.resolve](#path.resolve) + - [path.resolve_sibling](#path.resolve_sibling) + - [PathAttributes](#pathattributes) + - [SetReviewInput](#setreviewinput) + - [TransformWork](#transformwork) + - [ctx.add_label](#ctx.add_label) + - [ctx.add_or_replace_label](#ctx.add_or_replace_label) + - [ctx.add_text_before_labels](#ctx.add_text_before_labels) + - [ctx.create_symlink](#ctx.create_symlink) + - [ctx.destination_api](#ctx.destination_api) + - [ctx.destination_reader](#ctx.destination_reader) + - [ctx.find_all_labels](#ctx.find_all_labels) + - [ctx.find_label](#ctx.find_label) + - [ctx.new_path](#ctx.new_path) + - [ctx.now_as_string](#ctx.now_as_string) + - [ctx.origin_api](#ctx.origin_api) + - [ctx.read_path](#ctx.read_path) + - [ctx.remove_label](#ctx.remove_label) + - [ctx.replace_label](#ctx.replace_label) + - [ctx.run](#ctx.run) + - [ctx.set_author](#ctx.set_author) + - [ctx.set_message](#ctx.set_message) + - [ctx.write_path](#ctx.write_path) + + + +## author + +Represents the author of a change + + +#### Fields: + +Name | Description +---- | ----------- +email | The email of the author +name | The name of the author + + + +## authoring + +The authors mapping between an origin and a destination + + +### authoring.overwrite + +Use the default author for all the submits in the destination. Note that some destinations might choose to ignore this author and use the current user running the tool (In other words they don't allow impersonation). + +`authoring_class authoring.overwrite(default)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +default | `string`

The default author for commits in the destination

+ + +#### Example: + + +##### Overwrite usage example: + +Create an authoring object that will overwrite any origin author with noreply@foobar.com mail. + +```python +authoring.overwrite("Foo Bar ") +``` + + + +### authoring.pass_thru + +Use the origin author as the author in the destination, no whitelisting. + +`authoring_class authoring.pass_thru(default)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +default | `string`

The default author for commits in the destination. This is used in squash mode workflows or if author cannot be determined.

+ + +#### Example: + + +##### Pass thru usage example: + + + +```python +authoring.pass_thru(default = "Foo Bar ") +``` + + + +### authoring.whitelisted + +Create an individual or team that contributes code. + +`authoring_class authoring.whitelisted(default, whitelist)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +default | `string`

The default author for commits in the destination. This is used in squash mode workflows or when users are not whitelisted.

+whitelist | `sequence of string`

List of white listed authors in the origin. The authors must be unique

+ + +#### Examples: + + +##### Only pass thru whitelisted users: + + + +```python +authoring.whitelisted( + default = "Foo Bar ", + whitelist = [ + "someuser@myorg.com", + "other@myorg.com", + "another@myorg.com", + ], +) +``` + + +##### Only pass thru whitelisted LDAPs/usernames: + +Some repositories are not based on email but use LDAPs/usernames. This is also supported since it is up to the origin how to check whether two authors are the same. + +```python +authoring.whitelisted( + default = "Foo Bar ", + whitelist = [ + "someuser", + "other", + "another", + ], +) +``` + + + + +## authoring_class + +The authors mapping between an origin and a destination + + + +## buildozer + +Module for Buildozer-related functionality such as creating and modifying BUILD targets. + + +### buildozer.cmd + +Creates a Buildozer command. You can specify the reversal with the 'reverse' argument. + +`command buildozer.cmd(forward, reverse=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +forward | `string`

Specifies the Buildozer command, e.g. 'replace deps :foo :bar'

+reverse | `string`

The reverse of the command. This is only required if the given command cannot be reversed automatically and the reversal of this command is required by some workflow or Copybara check. The following commands are automatically reversible:

  • add
  • remove (when used to remove element from list i.e. 'remove srcs foo.cc'
  • replace

+ + +### buildozer.create + +A transformation which creates a new build target and populates its attributes. This transform can reverse automatically to delete the target. + +`buildozerCreate buildozer.create(target, rule_type, commands=[], before='', after='')` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +target | `string`

Target to create, including the package, e.g. 'foo:bar'. The package can be '.' for the root BUILD file.

+rule_type | `string`

Type of this rule, for instance, java_library.

+commands | `sequence`

Commands to populate attributes of the target after creating it. Elements can be strings such as 'add deps :foo' or objects returned by buildozer.cmd.

+before | `string`

When supplied, causes this target to be created *before* the target named by 'before'

+after | `string`

When supplied, causes this target to be created *after* the target named by 'after'

+ + +### buildozer.delete + +A transformation which is the opposite of creating a build target. When run normally, it deletes a build target. When reversed, it creates and prepares one. + +`buildozerDelete buildozer.delete(target, rule_type='', recreate_commands=[], before='', after='')` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +target | `string`

Target to create, including the package, e.g. 'foo:bar'

+rule_type | `string`

Type of this rule, for instance, java_library. Supplying this will cause this transformation to be reversible.

+recreate_commands | `sequence`

Commands to populate attributes of the target after creating it. Elements can be strings such as 'add deps :foo' or objects returned by buildozer.cmd.

+before | `string`

When supplied with rule_type and the transformation is reversed, causes this target to be created *before* the target named by 'before'

+after | `string`

When supplied with rule_type and the transformation is reversed, causes this target to be created *after* the target named by 'after'

+ + +### buildozer.modify + +A transformation which runs one or more Buildozer commands against a single target expression. See http://go/buildozer for details on supported commands and target expression formats. + +`buildozerModify buildozer.modify(target, commands)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +target | `object`

Specifies the target(s) against which to apply the commands. Can be a list.

+commands | `sequence`

Commands to apply to the target(s) specified. Elements can be strings such as 'add deps :foo' or objects returned by buildozer.cmd.

+ + +#### Examples: + + +##### Add a setting to one target: + +Add "config = ':foo'" to foo/bar:baz: + +```python +buildozer.modify( + target = 'foo/bar:baz', + commands = [ + buildozer.cmd('set config ":foo"'), + ], +) +``` + + +##### Add a setting to several targets: + +Add "config = ':foo'" to foo/bar:baz and foo/bar:fooz: + +```python +buildozer.modify( + target = ['foo/bar:baz', 'foo/bar:fooz'], + commands = [ + buildozer.cmd('set config ":foo"'), + ], +) +``` + + + + +## change + +A change metadata. Contains information like author, change message or detected labels + + +#### Fields: + +Name | Description +---- | ----------- +author | The author of the change +date_time_iso_offset | Return a ISO offset date time. Example: 2011-12-03T10:15:30+01:00' +first_line_message | The message of the change +labels | A dictionary with the labels detected for the change. If the label is present multiple times it returns the last value. Note that this is a heuristic and it could include things that are not labels. +labels_all_values | A dictionary with the labels detected for the change. Note that the value is a collection of the values for each time the label was found. Use 'labels' instead if you are only interested in the last value. Note that this is a heuristic and it could include things that are not labels. +merge | Returns true if the change represents a merge +message | The message of the change +original_author | The author of the change before any mapping +ref | Origin reference ref + + + +## ChangeMessage + +Represents a well formed parsed change message with its associated labels. + + +#### Fields: + +Name | Description +---- | ----------- +first_line | First line of this message +text | The text description this message, not including the labels. + + +### message.label_values + +Returns a list of values associated with the label name. + +`sequence of string message.label_values(label_name)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +label_name | `string`

The label name.

+ + + +## Changes + +Data about the set of changes that are being migrated. Each change includes information like: original author, change message, labels, etc. You receive this as a field in TransformWork object for used defined transformations + + +#### Fields: + +Name | Description +---- | ----------- +current | List of changes that will be migrated +migrated | List of changes that where migrated in previous Copybara executions or if using ITERATIVE mode in previous iterations of this workflow. + + + +## Console + +A console that can be used in skylark transformations to print info, warning or error messages. + + +### console.error + +Show an error in the log. Note that this will stop Copybara execution. + +`console.error(message)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +message | `string`

message to log

+ + +### console.info + +Show an info message in the console + +`console.info(message)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +message | `string`

message to log

+ + +### console.progress + +Show a progress message in the console + +`console.progress(message)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +message | `string`

message to log

+ + +### console.verbose + +Show an info message in the console if verbose logging is enabled. + +`console.verbose(message)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +message | `string`

message to log

+ + +### console.warn + +Show a warning in the console + +`console.warn(message)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +message | `string`

message to log

+ + + +## core + +Core functionality for creating migrations, and basic transformations. + + +#### Fields: + +Name | Description +---- | ----------- +main_config_path | Location of the config file. This is subject to change + + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--config-root` | *string* | Configuration root path to be used for resolving absolute config labels like '//foo/bar' +`--console-file-flush-interval` | *duration* | How often Copybara should flush the console to the output file. (10s, 1m, etc.)If set to 0s, console will be flushed only at the end. +`--console-file-path` | *string* | If set, write the console output also to the given file path. +`--debug-file-break` | *string* | Stop when file matching the glob changes +`--debug-metadata-break` | *boolean* | Stop when message and/or author changes +`--debug-transform-break` | *string* | Stop when transform description matches +`--disable-reversible-check` | *boolean* | If set, all workflows will be executed without reversible_check, overriding the workflow config and the normal behavior for CHANGE_REQUEST mode. +`--dry-run` | *boolean* | Run the migration in dry-run mode. Some destination implementations might have some side effects (like creating a code review), but never submit to a main branch. +`--fetch-timeout` | *duration* | Fetch timeout +`--force` | *boolean* | Force the migration even if Copybara cannot find in the destination a change that is an ancestor of the one(s) being migrated. This should be used with care, as it could lose changes when migrating a previous/conflicting change. +`--noansi` | *boolean* | Don't use ANSI output for messages +`--nocleanup` | *boolean* | Cleanup the output directories. This includes the workdir, scratch clones of Git repos, etc. By default is set to false and directories will be cleaned prior to the execution. If set to true, the previous run output will not be cleaned up. Keep in mind that running in this mode will lead to an ever increasing disk usage. +`--output-limit` | *int* | Limit the output in the console to a number of records. Each subcommand might use this flag differently. Defaults to 0, which shows all the output. +`--output-root` | *string* | The root directory where to generate output files. If not set, ~/copybara/out is used by default. Use with care, Copybara might remove files inside this root if necessary. +`--squash` | *boolean* | Override workflow's mode with 'SQUASH'. This is useful mainly for workflows that use 'ITERATIVE' mode, when we want to run a single export with 'SQUASH', maybe to fix an issue. Always use --dry-run before, to test your changes locally. +`--validate-starlark` | *string* | Starlark should be validated prior to execution, but this might break legacy configs. Options are LOOSE, STRICT +`-v, --verbose` | *boolean* | Verbose output. + + +### core.copy + +Copy files between directories and renames files + +`transformation core.copy(before, after, paths=glob(["**"]), overwrite=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +before | `string`

The name of the file or directory to copy. If this is the empty string and 'after' is a directory, then all files in the workdir will be copied to the sub directory specified by 'after', maintaining the directory tree.

+after | `string`

The name of the file or directory destination. If this is the empty string and 'before' is a directory, then all files in 'before' will be copied to the repo root, maintaining the directory tree inside 'before'.

+paths | `glob`

A glob expression relative to 'before' if it represents a directory. Only files matching the expression will be copied. For example, glob(["**.java"]), matches all java files recursively inside 'before' folder. Defaults to match all the files recursively.

+overwrite | `boolean`

Overwrite destination files if they already exist. Note that this makes the transformation non-reversible, since there is no way to know if the file was overwritten or not in the reverse workflow.

+ + +#### Examples: + + +##### Copy a directory: + +Move all the files in a directory to another directory: + +```python +core.copy("foo/bar_internal", "bar") +``` + +In this example, `foo/bar_internal/one` will be copied to `bar/one`. + + +##### Copy with reversal: + +Copy all static files to a 'static' folder and use remove for reverting the change + +```python +core.transform( + [core.copy("foo", "foo/static", paths = glob(["**.css","**.html", ]))], + reversal = [core.remove(glob(['foo/static/**.css', 'foo/static/**.html']))] +) +``` + + + +### core.dynamic_feedback + +Create a dynamic Skylark feedback migration. This should only be used by libraries developers + +`feedback.action core.dynamic_feedback(impl, params={})` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +impl | `starlarkCallable`

The Skylark function to call

+params | `dict`

The parameters to the function. Will be available under ctx.params

+ + +### core.dynamic_transform + +Create a dynamic Skylark transformation. This should only be used by libraries developers + +`transformation core.dynamic_transform(impl, params={})` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +impl | `starlarkCallable`

The Skylark function to call

+params | `dict`

The parameters to the function. Will be available under ctx.params

+ + +#### Example: + + +##### Create a dynamic transformation with parameter: + +If you want to create a library that uses dynamic transformations, you probably want to make them customizable. In order to do that, in your library.bara.sky, you need to hide the dynamic transformation (prefix with '_' and instead expose a function that creates the dynamic transformation with the param: + +```python +def _test_impl(ctx): + ctx.set_message(ctx.message + ctx.params['name'] + str(ctx.params['number']) + '\n') + +def test(name, number = 2): + return core.dynamic_transform(impl = _test_impl, + params = { 'name': name, 'number': number}) +``` + +After defining this function, you can use `test('example', 42)` as a transformation in `core.workflow`. + + + +### core.fail_with_noop + +If invoked, it will fail the current migration as a noop + +`feedback.action core.fail_with_noop(msg)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +msg | `string`

The noop message

+ + +### core.feedback + +Defines a migration of changes' metadata, that can be invoked via the Copybara command in the same way as a regular workflow migrates the change itself. + +It is considered change metadata any information associated with a change (pending or submitted) that is not core to the change itself. A few examples: +
    +
  • Comments: Present in any code review system. Examples: Github PRs or Gerrit code reviews.
    • +
  • Labels: Used in code review systems for approvals and/or CI results. Examples: Github labels, Gerrit code review labels.
    • +
+For the purpose of this workflow, it is not considered metadata the commit message in Git, or any of the contents of the file tree. + + + +`core.feedback(name, origin, destination, actions=[], description=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +name | `string`

The name of the feedback workflow.

+origin | `trigger`

The trigger of a feedback migration.

+destination | `endpoint_provider`

Where to write change metadata to. This is usually a code review system like Gerrit or GitHub PR.

+actions | `sequence`

A list of feedback actions to perform, with the following semantics:
- There is no guarantee of the order of execution.
- Actions need to be independent from each other.
- Failure in one action might prevent other actions from executing.

+description | `string`

A description of what this workflow achieves

+ + +### core.filter_replace + +Applies an initial filtering to find a substring to be replaced and then applies a `mapping` of replaces for the matched text. + +`filter_replace core.filter_replace(regex, mapping={}, group=Whole text, paths=glob(["**"]), reverse=`regex`)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +regex | `string`

A re2 regex to match a substring of the file

+mapping | `object`

A mapping function like core.replace_mapper or a dict with mapping values.

+group | `integer`

Extract a regex group from the matching text and pass this as parameter to the mapping instead of the whole matching text.

+paths | `glob`

A glob expression relative to the workdir representing the files to apply the transformation. For example, glob(["**.java"]), matches all java files recursively. Defaults to match all the files recursively.

+reverse | `string`

A re2 regex used as reverse transformation

+ + +#### Examples: + + +##### Simple replace with mapping: + +Simplest mapping + +```python +core.filter_replace( + regex = 'a.*', + mapping = { + 'afoo': 'abar', + 'abaz': 'abam' + } +) +``` + + +##### TODO replace: + +This replace is similar to what it can be achieved with core.todo_replace: + +```python +core.filter_replace( + regex = 'TODO\((.*?)\)', + group = 1, + mapping = core.replace_mapper([ + core.replace( + before = '${p}foo${s}', + after = '${p}fooz${s}', + regex_groups = { 'p': '.*', 's': '.*'} + ), + core.replace( + before = '${p}baz${s}', + after = '${p}bazz${s}', + regex_groups = { 'p': '.*', 's': '.*'} + ), + ], + all = True + ) +) +``` + + + +### core.format + +Formats a String using Java format patterns. + +`string core.format(format, args)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +format | `string`

The format string

+args | `sequence`

The arguments to format

+ + +### core.move + +Moves files between directories and renames files + +`transformation core.move(before, after, paths=glob(["**"]), overwrite=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +before | `string`

The name of the file or directory before moving. If this is the empty string and 'after' is a directory, then all files in the workdir will be moved to the sub directory specified by 'after', maintaining the directory tree.

+after | `string`

The name of the file or directory after moving. If this is the empty string and 'before' is a directory, then all files in 'before' will be moved to the repo root, maintaining the directory tree inside 'before'.

+paths | `glob`

A glob expression relative to 'before' if it represents a directory. Only files matching the expression will be moved. For example, glob(["**.java"]), matches all java files recursively inside 'before' folder. Defaults to match all the files recursively.

+overwrite | `boolean`

Overwrite destination files if they already exist. Note that this makes the transformation non-reversible, since there is no way to know if the file was overwritten or not in the reverse workflow.

+ + +#### Examples: + + +##### Move a directory: + +Move all the files in a directory to another directory: + +```python +core.move("foo/bar_internal", "bar") +``` + +In this example, `foo/bar_internal/one` will be moved to `bar/one`. + + +##### Move all the files to a subfolder: + +Move all the files in the checkout dir into a directory called foo: + +```python +core.move("", "foo") +``` + +In this example, `one` and `two/bar` will be moved to `foo/one` and `foo/two/bar`. + + +##### Move a subfolder's content to the root: + +Move the contents of a folder to the checkout root directory: + +```python +core.move("foo", "") +``` + +In this example, `foo/bar` would be moved to `bar`. + + + +### core.remove + +Remove files from the workdir. **This transformation is only meant to be used inside core.transform for reversing core.copy like transforms**. For regular file filtering use origin_files exclude mechanism. + +`remove core.remove(paths)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +paths | `glob`

The files to be deleted

+ + +#### Examples: + + +##### Reverse a file copy: + +Move all the files in a directory to another directory: + +```python +core.transform( + [core.copy("foo", "foo/public")], + reversal = [core.remove(glob(["foo/public/**"]))]) +``` + +In this example, `foo/one` will be moved to `foo/public/one`. + + +##### Copy with reversal: + +Copy all static files to a 'static' folder and use remove for reverting the change + +```python +core.transform( + [core.copy("foo", "foo/static", paths = glob(["**.css","**.html", ]))], + reversal = [core.remove(glob(['foo/static/**.css', 'foo/static/**.html']))] +) +``` + + + +### core.replace + +Replace a text with another text using optional regex groups. This tranformer can be automatically reversed. + +`replace core.replace(before, after, regex_groups={}, paths=glob(["**"]), first_only=False, multiline=False, repeated_groups=False, ignore=[])` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +before | `string`

The text before the transformation. Can contain references to regex groups. For example "foo${x}text".

`before` can only contain 1 reference to each unique `regex_group`. If you require multiple references to the same `regex_group`, add `repeated_groups: True`.

If '$' literal character needs to be matched, '`$$`' should be used. For example '`$$FOO`' would match the literal '$FOO'.

+after | `string`

The text after the transformation. It can also contain references to regex groups, like 'before' field.

+regex_groups | `dict`

A set of named regexes that can be used to match part of the replaced text.Copybara uses [re2](https://github.com/google/re2/wiki/Syntax) syntax. For example {"x": "[A-Za-z]+"}

+paths | `glob`

A glob expression relative to the workdir representing the files to apply the transformation. For example, glob(["**.java"]), matches all java files recursively. Defaults to match all the files recursively.

+first_only | `boolean`

If true, only replaces the first instance rather than all. In single line mode, replaces the first instance on each line. In multiline mode, replaces the first instance in each file.

+multiline | `boolean`

Whether to replace text that spans more than one line.

+repeated_groups | `boolean`

Allow to use a group multiple times. For example foo${repeated}/${repeated}. Note that this mechanism doesn't use backtracking. In other words, the group instances are treated as different groups in regex construction and then a validation is done after that.

+ignore | `sequence`

A set of regexes. Any text that matches any expression in this set, which might otherwise be transformed, will be ignored.

+ + +#### Examples: + + +##### Simple replacement: + +Replaces the text "internal" with "external" in all java files + +```python +core.replace( + before = "internal", + after = "external", + paths = glob(["**.java"]), +) +``` + + +##### Append some text at the end of files: + + + +```python +core.replace( + before = '${end}', + after = 'Text to be added at the end', + multiline = True, + regex_groups = { 'end' : '\z'}, +) +``` + + +##### Append some text at the end of files reversible: + +Same as the above example but make the transformation reversible + +```python +core.transform([ + core.replace( + before = '${end}', + after = 'some append', + multiline = True, + regex_groups = { 'end' : '\z'}, + ) +], +reversal = [ + core.replace( + before = 'some append${end}', + after = '', + multiline = True, + regex_groups = { 'end' : '\z'}, + )]) +``` + + +##### Replace using regex groups: + +In this example we map some urls from the internal to the external version in all the files of the project. + +```python +core.replace( + before = "https://some_internal/url/${pkg}.html", + after = "https://example.com/${pkg}.html", + regex_groups = { + "pkg": ".*", + }, + ) +``` + +So a url like `https://some_internal/url/foo/bar.html` will be transformed to `https://example.com/foo/bar.html`. + + +##### Remove confidential blocks: + +This example removes blocks of text/code that are confidential and thus shouldn'tbe exported to a public repository. + +```python +core.replace( + before = "${x}", + after = "", + multiline = True, + regex_groups = { + "x": "(?m)^.*BEGIN-INTERNAL[\\w\\W]*?END-INTERNAL.*$\\n", + }, + ) +``` + +This replace would transform a text file like: + +``` +This is +public + // BEGIN-INTERNAL + confidential + information + // END-INTERNAL +more public code + // BEGIN-INTERNAL + more confidential + information + // END-INTERNAL +``` + +Into: + +``` +This is +public +more public code +``` + + + + + +### core.replace_mapper + +A mapping function that applies a list of replaces until one replaces the text (Unless `all = True` is used). This should be used with core.filter_replace or other transformations that accept text mapping as parameter. + +`replaceMapper core.replace_mapper(mapping, all=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +mapping | `sequence of transformation`

The list of core.replace transformations

+all | `boolean`

Run all the mappings despite a replace happens.

+ + +### core.reverse + +Given a list of transformations, returns the list of transformations equivalent to undoing all the transformations + +`sequence of transformation core.reverse(transformations)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +transformations | `sequence of transformation`

The transformations to reverse

+ + +### core.todo_replace + +Replace Google style TODOs. For example `TODO(username, othername)`. + +`todoReplace core.todo_replace(tags=['TODO', 'NOTE'], mapping={}, mode='MAP_OR_IGNORE', paths=glob(["**"]), default=None, ignore=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +tags | `sequence of string`

Prefix tag to look for

+mapping | `dict`

Mapping of users/strings

+mode | `string`

Mode for the replace:

  • 'MAP_OR_FAIL': Try to use the mapping and if not found fail.
  • 'MAP_OR_IGNORE': Try to use the mapping but ignore if no mapping found.
  • 'MAP_OR_DEFAULT': Try to use the mapping and use the default if not found.
  • 'SCRUB_NAMES': Scrub all names from TODOs. Transforms 'TODO(foo)' to 'TODO'
  • 'USE_DEFAULT': Replace any TODO(foo, bar) with TODO(default_string)

+paths | `glob`

A glob expression relative to the workdir representing the files to apply the transformation. For example, glob(["**.java"]), matches all java files recursively. Defaults to match all the files recursively.

+default | `string`

Default value if mapping not found. Only valid for 'MAP_OR_DEFAULT' or 'USE_DEFAULT' modes

+ignore | `string`

If set, elements within TODO (with usernames) that match the regex will be ignored. For example ignore = "foo" would ignore "foo" in "TODO(foo,bar)" but not "bar".

+ + +#### Examples: + + +##### Simple update: + +Replace TODOs and NOTES for users in the mapping: + +```python +core.todo_replace( + mapping = { + 'test1' : 'external1', + 'test2' : 'external2' + } +) +``` + +Would replace texts like TODO(test1) or NOTE(test1, test2) with TODO(external1) or NOTE(external1, external2) + + +##### Scrubbing: + +Remove text from inside TODOs + +```python +core.todo_replace( + mode = 'SCRUB_NAMES' +) +``` + +Would replace texts like TODO(test1): foo or NOTE(test1, test2):foo with TODO:foo and NOTE:foo + + +##### Ignoring Regex Patterns: + +Ignore regEx inside TODOs when scrubbing/mapping + +```python +core.todo_replace( + mapping = { 'aaa' : 'foo'}, + ignore = 'b/.*' +) +``` + +Would replace texts like TODO(b/123, aaa) with TODO(b/123, foo) + + + +### core.transform + +Groups some transformations in a transformation that can contain a particular, manually-specified, reversal, where the forward version and reversed version of the transform are represented as lists of transforms. The is useful if a transformation does not automatically reverse, or if the automatic reversal does not work for some reason.
If reversal is not provided, the transform will try to compute the reverse of the transformations list. + +`transformation core.transform(transformations, reversal=The reverse of 'transformations', ignore_noop=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +transformations | `sequence of transformation`

The list of transformations to run as a result of running this transformation.

+reversal | `sequence of transformation`

The list of transformations to run as a result of running this transformation in reverse.

+ignore_noop | `boolean`

In case a noop error happens in the group of transformations (Both forward and reverse), it will be ignored, but the rest of the transformations in the group will still be executed. If ignore_noop is not set, we will apply the closest parent's ignore_noop.

+ + +### core.verify_match + +Verifies that a RegEx matches (or not matches) the specified files. Does not transform anything, but will stop the workflow if it fails. + +`verifyMatch core.verify_match(regex, paths=glob(["**"]), verify_no_match=False, also_on_reversal=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +regex | `string`

The regex pattern to verify. To satisfy the validation, there has to be atleast one (or no matches if verify_no_match) match in each of the files included in paths. The re2j pattern will be applied in multiline mode, i.e. '^' refers to the beginning of a file and '$' to its end. Copybara uses [re2](https://github.com/google/re2/wiki/Syntax) syntax.

+paths | `glob`

A glob expression relative to the workdir representing the files to apply the transformation. For example, glob(["**.java"]), matches all java files recursively. Defaults to match all the files recursively.

+verify_no_match | `boolean`

If true, the transformation will verify that the RegEx does not match.

+also_on_reversal | `boolean`

If true, the check will also apply on the reversal. The default behavior is to not verify the pattern on reversal.

+ + +### core.workflow + +Defines a migration pipeline which can be invoked via the Copybara command. + +Implicit labels that can be used/exposed: + + - COPYBARA_CONTEXT_REFERENCE: Requested reference. For example if copybara is invoked as `copybara copy.bara.sky workflow master`, the value would be `master`. + - COPYBARA_LAST_REV: Last reference that was migrated + - COPYBARA_CURRENT_REV: The current reference being migrated + - COPYBARA_CURRENT_MESSAGE: The current message at this point of the transformations + - COPYBARA_CURRENT_MESSAGE_TITLE: The current message title (first line) at this point of the transformations + - COPYBARA_AUTHOR: The author of the change + + +`core.workflow(name, origin, destination, authoring, transformations=[], origin_files=glob(["**"]), destination_files=glob(["**"]), mode="SQUASH", reversible_check=True for 'CHANGE_REQUEST' mode. False otherwise, check_last_rev_state=True for CHANGE_REQUEST, ask_for_confirmation=False, dry_run=False, after_migration=[], after_workflow=[], change_identity=None, set_rev_id=True, smart_prune=False, migrate_noop_changes=False, experimental_custom_rev_id=None, description=None, checkout=True)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +name | `string`

The name of the workflow.

+origin | `origin`

Where to read from the code to be migrated, before applying the transformations. This is usually a VCS like Git, but can also be a local folder or even a pending change in a code review system like Gerrit.

+destination | `destination`

Where to write to the code being migrated, after applying the transformations. This is usually a VCS like Git, but can also be a local folder or even a pending change in a code review system like Gerrit.

+authoring | `authoring_class`

The author mapping configuration from origin to destination.

+transformations | `sequence`

The transformations to be run for this workflow. They will run in sequence.

+origin_files | `glob`

A glob relative to the workdir that will be read from the origin during the import. For example glob(["**.java"]), all java files, recursively, which excludes all other file types.

+destination_files | `glob`

A glob relative to the root of the destination repository that matches files that are part of the migration. Files NOT matching this glob will never be removed, even if the file does not exist in the source. For example glob(['**'], exclude = ['**/BUILD']) keeps all BUILD files in destination when the origin does not have any BUILD files. You can also use this to limit the migration to a subdirectory of the destination, e.g. glob(['java/src/**'], exclude = ['**/BUILD']) to only affect non-BUILD files in java/src.

+mode | `string`

Workflow mode. Currently we support four modes:

  • 'SQUASH': Create a single commit in the destination with new tree state.
  • 'ITERATIVE': Import each origin change individually.
  • 'CHANGE_REQUEST': Import a pending change to the Source-of-Truth. This could be a GH Pull Request, a Gerrit Change, etc. The final intention should be to submit the change in the SoT (destination in this case).
  • 'CHANGE_REQUEST_FROM_SOT': Import a pending change **from** the Source-of-Truth. This mode is useful when, despite the pending change being already in the SoT, the users want to review the code on a different system. The final intention should never be to submit in the destination, but just review or test

+reversible_check | `boolean`

Indicates if the tool should try to to reverse all the transformations at the end to check that they are reversible.
The default value is True for 'CHANGE_REQUEST' mode. False otherwise

+check_last_rev_state | `boolean`

If set to true, Copybara will validate that the destination didn't change since last-rev import for destination_files. Note that this flag doesn't work for CHANGE_REQUEST mode.

+ask_for_confirmation | `boolean`

Indicates that the tool should show the diff and require user's confirmation before making a change in the destination.

+dry_run | `boolean`

Run the migration in dry-run mode. Some destination implementations might have some side effects (like creating a code review), but never submit to a main branch.

+after_migration | `sequence`

Run a feedback workflow after one migration happens. This runs once per change in `ITERATIVE` mode and only once for `SQUASH`.

+after_workflow | `sequence`

Run a feedback workflow after all the changes for this workflow run are migrated. Prefer `after_migration` as it is executed per change (in ITERATIVE mode). Tasks in this hook shouldn't be critical to execute. These actions shouldn't record effects (They'll be ignored).

+change_identity | `string`

By default, Copybara hashes several fields so that each change has an unique identifier that at the same time reuses the generated destination change. This allows to customize the identity hash generation so that the same identity is used in several workflows. At least ${copybara_config_path} has to be present. Current user is added to the hash automatically.

Available variables:

  • ${copybara_config_path}: Main config file path
  • ${copybara_workflow_name}: The name of the workflow being run
  • ${copybara_reference}: The requested reference. In general Copybara tries its best to give a repetable reference. For example Gerrit change number or change-id or GitHub Pull Request number. If it cannot find a context reference it uses the resolved revision.
  • ${label:label_name}: A label present for the current change. Exposed in the message or not.
If any of the labels cannot be found it defaults to the default identity (The effect would be no reuse of destination change between workflows)

+set_rev_id | `boolean`

Copybara adds labels like 'GitOrigin-RevId' in the destination in order to track what was the latest change imported. For `CHANGE_REQUEST` workflows it is not used and is purely informational. This field allows to disable it for that mode. Destinations might ignore the flag.

+smart_prune | `boolean`

By default CHANGE_REQUEST workflows cannot restore scrubbed files. This flag does a best-effort approach in restoring the non-affected snippets. For now we only revert the non-affected files. This only works for CHANGE_REQUEST mode.

+migrate_noop_changes | `boolean`

By default, Copybara tries to only migrate changes that affect origin_files or config files. This flag allows to include all the changes. Note that it might generate more empty changes errors. In `ITERATIVE` mode it might fail if some transformation is validating the message (Like has to contain 'PUBLIC' and the change doesn't contain it because it is internal).

+experimental_custom_rev_id | `string`

Use this label name instead of the one provided by the origin. This is subject to change and there is no guarantee.

+description | `string`

A description of what this workflow achieves

+checkout | `boolean`

Allows disabling the checkout. The usage of this feature is rare. This could be used to update a file of your own repo when a dependant repo version changes and you are not interested on the files of the dependant repo, just the new version.

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--change-request-from-sot-limit` | *int* | Number of origin baseline changes to use for trying to match one in the destination. It can be used if the are many parent changes in the origin that are a no-op in the destination +`--change-request-from-sot-retry` | *list<integer>* | Number of retries and delay between retries when we cannot find the baseline in the destination for CHANGE_REQUEST_FROM_SOT. For example '10,30,60' will retry three times. The first retry will be delayed 10s, the second one 30s and the third one 60s +`--change-request-parent, --change_request_parent` | *string* | Commit revision to be used as parent when importing a commit using CHANGE_REQUEST workflow mode. this shouldn't be needed in general as Copybara is able to detect the parent commit message. +`--check-last-rev-state` | *boolean* | If enabled, Copybara will validate that the destination didn't change since last-rev import for destination_files. Note that this flag doesn't work for CHANGE_REQUEST mode. +`--default-author` | *string* | Use this author as default instead of the one in the config file.Format should be 'Foo Bar ' +`--diff-in-origin` | *boolean* | When this flag is enabled, copybara will show different changes between last Revision and current revision in origin instead of in destination. NOTE: it Only works for SQUASH and ITERATIVE +`--force-author` | *author* | Force the author to this. Note that this only changes the author before the transformations happen, you can still use the transformations to alter it. +`--force-message` | *string* | Force the change description to this. Note that this only changes the message before the transformations happen, you can still use the transformations to alter it. +`--ignore-noop` | *boolean* | Only warn about operations/transforms that didn't have any effect. For example: A transform that didn't modify any file, non-existent origin directories, etc. +`--import-noop-changes` | *boolean* | By default Copybara will only try to migrate changes that could affect the destination. Ignoring changes that only affect excluded files in origin_files. This flag disables that behavior and runs for all the changes. +`--init-history` | *boolean* | Import all the changes from the beginning of the history up to the resolved ref. For 'ITERATIVE' workflows this will import individual changes since the first one. For 'SQUASH' it will import the squashed change up to the resolved ref. WARNING: Use with care, this flag should be used only for the very first run of Copybara for a workflow. +`--iterative-limit-changes` | *int* | Import just a number of changes instead of all the pending ones +`--last-rev` | *string* | Last revision that was migrated to the destination +`--nosmart-prune` | *boolean* | Disable smart prunning +`--notransformation-join` | *boolean* | By default Copybara tries to join certain transformations in one so that it is more efficient. This disables the feature. +`--read-config-from-change` | *boolean* | For each imported origin change, load the workflow's origin_files, destination_files and transformations from the config version of that change. The rest of the fields (more importantly, origin and destination) cannot change and the version from the first config will be used. +`--squash-skip-history` | *boolean* | Avoid exposing the history of changes that are being migrated. This is useful when we want to migrate a new repository but we don't want to expose all the change history to metadata.squash_notes. +`--threads` | *int* | Number of threads to use when running transformations that change lot of files +`--threads-min-size` | *int* | Minimum size of the lists to process to run them in parallel +`--workflow-identity-user` | *string* | Use a custom string as a user for computing change identity + + + +## destination_effect + +Represents an effect that happened in the destination due to a single migration + + +#### Fields: + +Name | Description +---- | ----------- +destination_ref | Destination reference updated/created. Might be null if there was no effect. Might be set even if the type is error (For example a synchronous presubmit test failed but a review was created). +errors | List of errors that happened during the migration +origin_refs | List of origin changes that were included in this migration +summary | Textual summary of what happened. Users of this class should not try to parse this field. +type | Return the type of effect that happened: CREATED, UPDATED, NOOP, INSUFFICIENT_APPROVALS or ERROR + + + +## destination_reader + +Handle to read from the destination + + +### destination_reader.copy_destination_files + +Copy files from the destination into the workdir. + +`destination_reader.copy_destination_files(glob)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +glob | `glob`

Files to copy to the workdir, potentially overwriting files checked out from the origin.

+ + +#### Example: + + +##### Copy files from the destination's baseline: + +This can be added to the transformations of your core.workflow: + +```python +def _copy_destination_file(ctx): + content = ctx.destination_reader().copy_destination_files(path = 'path/to/**') + + transforms = [core.dynamic_transform(_copy_destination_file)] + +``` + +Would copy all files in path/to/ from the destination baseline to the copybara workdir. The files do not have to be covered by origin_files nor destination_files, but will cause errors if they are not covered by destination_files and not moved or deleted. + + + +### destination_reader.read_file + +Read a file from the destination. + +`string destination_reader.read_file(path)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +path | `string`

Path to the file.

+ + +#### Example: + + +##### Read a file from the destination's baseline: + +This can be added to the transformations of your core.workflow: + +```python +def _read_destination_file(ctx): + content = ctx.destination_reader().read_file(path = path/to/my_file.txt') + ctx.console.info(content) + + transforms = [core.dynamic_transform(_read_destination_file)] + +``` + +Would print out the content of path/to/my_file.txt in the destination. The file does not have to be covered by origin_files nor destination_files. + + + + +## destination_ref + +Reference to the change/review created/updated on the destination. + + +#### Fields: + +Name | Description +---- | ----------- +id | Destination reference id +type | Type of reference created. Each destination defines its own and guarantees to be more stable than urls/ids +url | Url, if any, of the destination change + + + +## endpoint + +An origin or destination API in a feedback migration. + + +#### Fields: + +Name | Description +---- | ----------- +url | Return the URL of this endpoint. + + +### endpoint.new_destination_ref + +Creates a new destination reference out of this endpoint. + +`destination_ref endpoint.new_destination_ref(ref, type, url=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +ref | `string`

The reference.

+type | `string`

The type of this reference.

+url | `string`

The url associated with this reference, if any.

+ + +### endpoint.new_origin_ref + +Creates a new origin reference out of this endpoint. + +`origin_ref endpoint.new_origin_ref(ref)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +ref | `string`

The reference.

+ + + +## endpoint_provider + +An handle for an origin or destination API in a feedback migration. + + +#### Fields: + +Name | Description +---- | ----------- +url | Return the URL of this endpoint, if any. + + + +## feedback.action_result + +Gives access to the feedback migration information and utilities. + + +#### Fields: + +Name | Description +---- | ----------- +msg | The message associated with the result +result | The result of this action + + + +## feedback.context + +Gives access to the feedback migration information and utilities. This context is a concrete implementation for feedback migrations. + + +#### Fields: + +Name | Description +---- | ----------- +action_name | The name of the current action. +console | Get an instance of the console to report errors or warnings +destination | An object representing the destination. Can be used to query or modify the destination state +feedback_name | The name of the Feedback migration calling this action. +origin | An object representing the origin. Can be used to query about the ref or modifying the origin state +params | Parameters for the function if created with core.dynamic_feedback +refs | A list containing string representations of the entities that triggered the event + + +### feedback.context.error + +Returns an error action result. + +`feedback.action_result feedback.context.error(msg)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +msg | `string`

The error message

+ + +### feedback.context.noop + +Returns a no op action result with an optional message. + +`feedback.action_result feedback.context.noop(msg=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +msg | `string`

The no op message

+ + +### feedback.context.record_effect + +Records an effect of the current action. + +`feedback.context.record_effect(summary, origin_refs, destination_ref, errors=[], type="UPDATED")` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +summary | `string`

The summary of this effect

+origin_refs | `sequence of origin_ref`

The origin refs

+destination_ref | `destination_ref`

The destination ref

+errors | `sequence of string`

An optional list of errors

+type | `string`

The type of migration effect:

  • 'CREATED': A new review or change was created.
  • 'UPDATED': An existing review or change was updated.
  • 'NOOP': The change was a noop.
  • 'INSUFFICIENT_APPROVALS': The effect couldn't happen because the change doesn't have enough approvals.
  • 'ERROR': A user attributable error happened that prevented the destination from creating/updating the change.
  • 'STARTED': The initial effect of a migration that depends on a previous one. This allows to have 'dependant' migrations defined by users.
    An example of this: a workflow migrates code from a Gerrit review to a GitHub PR, and a feedback migration migrates the test results from a CI in GitHub back to the Gerrit change.
    This effect would be created on the former one.

+ + +### feedback.context.success + +Returns a successful action result. + +`feedback.action_result feedback.context.success()` + + + +## feedback.finish_hook_context + +Gives access to the feedback migration information and utilities. This context is a concrete implementation for 'after_migration' hooks. + + +#### Fields: + +Name | Description +---- | ----------- +action_name | The name of the current action. +console | Get an instance of the console to report errors or warnings +destination | An object representing the destination. Can be used to query or modify the destination state +effects | The list of effects that happened in the destination +origin | An object representing the origin. Can be used to query about the ref or modifying the origin state +params | Parameters for the function if created with core.dynamic_feedback +revision | Get the requested/resolved revision + + +### feedback.finish_hook_context.record_effect + +Records an effect of the current action. + +`feedback.finish_hook_context.record_effect(summary, origin_refs, destination_ref, errors=[], type="UPDATED")` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +summary | `string`

The summary of this effect

+origin_refs | `sequence of origin_ref`

The origin refs

+destination_ref | `destination_ref`

The destination ref

+errors | `sequence of string`

An optional list of errors

+type | `string`

The type of migration effect:

  • 'CREATED': A new review or change was created.
  • 'UPDATED': An existing review or change was updated.
  • 'NOOP': The change was a noop.
  • 'INSUFFICIENT_APPROVALS': The effect couldn't happen because the change doesn't have enough approvals.
  • 'ERROR': A user attributable error happened that prevented the destination from creating/updating the change.
  • 'STARTED': The initial effect of a migration that depends on a previous one. This allows to have 'dependant' migrations defined by users.
    An example of this: a workflow migrates code from a Gerrit review to a GitHub PR, and a feedback migration migrates the test results from a CI in GitHub back to the Gerrit change.
    This effect would be created on the former one.

+ + + +## feedback.revision_context + +Information about the revision request/resolved for the migration + + +#### Fields: + +Name | Description +---- | ----------- +labels | A dictionary with the labels detected for the requested/resolved revision. + + + +## filter_replace + +A core.filter_replace transformation + + + +## folder + +Module for dealing with local filesystem folders + + +### folder.destination + +A folder destination is a destination that puts the output in a folder. It can be used both for testing or real production migrations.Given that folder destination does not support a lot of the features of real VCS, there are some limitations on how to use it:
  • It requires passing a ref as an argument, as there is no way of calculating previous migrated changes. Alternatively, --last-rev can be used, which could migrate N changes.
  • Most likely, the workflow should use 'SQUASH' mode, as history is not supported.
  • If 'ITERATIVE' mode is used, a new temp directory will be created for each change migrated.
+ +`folderDestination folder.destination()` + + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--folder-dir` | *string* | Local directory to write the output of the migration to. If the directory exists, all files will be deleted. By default Copybara will generate a temporary directory, so you shouldn't need this. + + +### folder.origin + +A folder origin is a origin that uses a folder as input. The folder is specified via the source_ref argument. + +`folderOrigin folder.origin(materialize_outside_symlinks=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +materialize_outside_symlinks | `boolean`

By default folder.origin will refuse any symlink in the migration folder that is an absolute symlink or that refers to a file outside of the folder. If this flag is set, it will materialize those symlinks as regular files in the checkout directory.

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--folder-origin-author` | *string* | Deprecated. Please use '--force-author'. Author of the change being migrated from folder.origin() +`--folder-origin-ignore-invalid-symlinks` | *boolean* | If an invalid symlink is found, ignore it instead of failing +`--folder-origin-message` | *string* | Deprecated. Please use '--force-message'. Message of the change being migrated from folder.origin() + + + +## format + +Module for formatting the code to Google's style/guidelines + + +### format.buildifier + +Formats the BUILD files using buildifier. + +`transformation format.buildifier(paths=glob(["**.bzl", "**/BUILD", "BUILD"]), type='auto', lint="OFF", lint_warnings=[])` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +paths | `glob`

Paths of the files to format relative to the workdir.

+type | `string`

The type of the files. Can be 'auto', 'bzl', 'build' or 'workspace'. Note that this is not recommended to be set and might break in the future. The default is 'auto'. This mode formats as BUILD files "BUILD", "BUILD.bazel", "WORKSPACE" and "WORKSPACE.bazel" files. The rest as bzl files. Prefer to use those names for BUILD files instead of setting this flag.

+lint | `string`

If buildifier --lint should be used. This fixes several common issues. Note that this transformation is difficult to revert. For example if it removes a load statement because is not used after removing a rule, then the reverse workflow needs to add back the load statement (core.replace or similar). Possible values: `OFF`, `FIX`. Default is `OFF`

+lint_warnings | `sequence of string`

Warnings used in the lint mode. Default is buildifier default`

+ + +#### Examples: + + +##### Default usage: + +The default parameters formats all BUILD and bzl files in the checkout directory: + +```python +format.buildifier() +``` + + +##### Enable lint: + +Enable lint for buildifier + +```python +format.buildifier(lint = "FIX") +``` + + +##### Using globs: + +Globs can be used to match only certain files: + +```python +format.buildifier( + paths = glob(["foo/BUILD", "foo/**/BUILD"], exclude = ["foo/bar/BUILD"]) +) +``` + +Formats all the BUILD files inside `foo` except for `foo/bar/BUILD` + + + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--buildifier-batch-size` | *int* | Process files in batches this size + + + +## gerritapi.AccountInfo + +Gerrit account information. + + +#### Fields: + +Name | Description +---- | ----------- +account_id | The numeric ID of the account. +email | The email address the user prefers to be contacted through.
Only set if detailed account information is requested.
See option DETAILED_ACCOUNTS for change queries
and options DETAILS and ALL_EMAILS for account queries. +name | The full name of the user.
Only set if detailed account information is requested.
See option DETAILED_ACCOUNTS for change queries
and option DETAILS for account queries. +secondary_emails | A list of the secondary email addresses of the user.
Only set for account queries when the ALL_EMAILS option or the suggest parameter is set.
Secondary emails are only included if the calling user has the Modify Account, and hence is allowed to see secondary emails of other users. +username | The username of the user.
Only set if detailed account information is requested.
See option DETAILED_ACCOUNTS for change queries
and option DETAILS for account queries. + + + +## gerritapi.ApprovalInfo + +Gerrit approval information. + + +#### Fields: + +Name | Description +---- | ----------- +account_id | The numeric ID of the account. +date | The time and date describing when the approval was made. +email | The email address the user prefers to be contacted through.
Only set if detailed account information is requested.
See option DETAILED_ACCOUNTS for change queries
and options DETAILS and ALL_EMAILS for account queries. +name | The full name of the user.
Only set if detailed account information is requested.
See option DETAILED_ACCOUNTS for change queries
and option DETAILS for account queries. +secondary_emails | A list of the secondary email addresses of the user.
Only set for account queries when the ALL_EMAILS option or the suggest parameter is set.
Secondary emails are only included if the calling user has the Modify Account, and hence is allowed to see secondary emails of other users. +username | The username of the user.
Only set if detailed account information is requested.
See option DETAILED_ACCOUNTS for change queries
and option DETAILS for account queries. +value | The vote that the user has given for the label. If present and zero, the user is permitted to vote on the label. If absent, the user is not permitted to vote on that label. + + + +## gerritapi.ChangeInfo + +Gerrit change information. + + +#### Fields: + +Name | Description +---- | ----------- +branch | The name of the target branch.
The refs/heads/ prefix is omitted. +change_id | The Change-Id of the change. +created | The timestamp of when the change was created. +current_revision | The commit ID of the current patch set of this change.
Only set if the current revision is requested or if all revisions are requested. +id | The ID of the change in the format "`~~`", where 'project', 'branch' and 'Change-Id' are URL encoded. For 'branch' the refs/heads/ prefix is omitted. +labels | The labels of the change as a map that maps the label names to LabelInfo entries.
Only set if labels or detailed labels are requested. +messages | Messages associated with the change as a list of ChangeMessageInfo entities.
Only set if messages are requested. +number | The legacy numeric ID of the change. +owner | The owner of the change as an AccountInfo entity. +project | The name of the project. +revisions | All patch sets of this change as a map that maps the commit ID of the patch set to a RevisionInfo entity.
Only set if the current revision is requested (in which case it will only contain a key for the current revision) or if all revisions are requested. +status | The status of the change (NEW, MERGED, ABANDONED). +subject | The subject of the change (header line of the commit message). +submittable | Whether the change has been approved by the project submit rules. Only set if requested via additional field SUBMITTABLE. +submitted | The timestamp of when the change was submitted. +topic | The topic to which this change belongs. +updated | The timestamp of when the change was last updated. + + + +## gerritapi.ChangeMessageInfo + +Gerrit change message information. + + +#### Fields: + +Name | Description +---- | ----------- +author | Author of the message as an AccountInfo entity.
Unset if written by the Gerrit system. +date | The timestamp of when this identity was constructed. +id | The ID of the message. +message | The text left by the user. +real_author | Real author of the message as an AccountInfo entity.
Set if the message was posted on behalf of another user. +revision_number | Which patchset (if any) generated this message. +tag | Value of the tag field from ReviewInput set while posting the review. NOTE: To apply different tags on on different votes/comments multiple invocations of the REST call are required. + + + +## gerritapi.ChangesQuery + +Input for listing Gerrit changes. See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes + + + +## gerritapi.CommitInfo + +Gerrit commit information. + + +#### Fields: + +Name | Description +---- | ----------- +author | The author of the commit as a GitPersonInfo entity. +commit | The commit ID. Not set if included in a RevisionInfo entity that is contained in a map which has the commit ID as key. +committer | The committer of the commit as a GitPersonInfo entity. +message | The commit message. +parents | The parent commits of this commit as a list of CommitInfo entities. In each parent only the commit and subject fields are populated. +subject | The subject of the commit (header line of the commit message). + + + +## gerritapi.GitPersonInfo + +Git person information. + + +#### Fields: + +Name | Description +---- | ----------- +date | The timestamp of when this identity was constructed. +email | The email address of the author/committer. +name | The name of the author/committer. + + + +## gerritapi.LabelInfo + +Gerrit label information. + + +#### Fields: + +Name | Description +---- | ----------- +all | List of all approvals for this label as a list of ApprovalInfo entities. Items in this list may not represent actual votes cast by users; if a user votes on any label, a corresponding ApprovalInfo will appear in this list for all labels. +approved | One user who approved this label on the change (voted the maximum value) as an AccountInfo entity. +blocking | If true, the label blocks submit operation. If not set, the default is false. +default_value | The default voting value for the label. This value may be outside the range specified in permitted_labels. +disliked | One user who disliked this label on the change (voted negatively, but not the minimum value) as an AccountInfo entity. +recommended | One user who recommended this label on the change (voted positively, but not the maximum value) as an AccountInfo entity. +rejected | One user who rejected this label on the change (voted the minimum value) as an AccountInfo entity. +value | The voting value of the user who recommended/disliked this label on the change if it is not “+1”/“-1”. +values | A map of all values that are allowed for this label. The map maps the values (“-2”, “-1”, " `0`", “+1”, “+2”) to the value descriptions. + + + +## gerritapi.ParentCommitInfo + +Gerrit parent commit information. + + +#### Fields: + +Name | Description +---- | ----------- +commit | The commit ID. Not set if included in a RevisionInfo entity that is contained in a map which has the commit ID as key. +subject | The subject of the commit (header line of the commit message). + + + +## gerritapi.ReviewResult + +Gerrit review result. + + +#### Fields: + +Name | Description +---- | ----------- +labels | Map of labels to values after the review was posted. +ready | If true, the change was moved from WIP to ready for review as a result of this action. Not set if false. + + + +## gerritapi.RevisionInfo + +Gerrit revision information. + + +#### Fields: + +Name | Description +---- | ----------- +commit | The commit of the patch set as CommitInfo entity. +created | The timestamp of when the patch set was created. +kind | The change kind. Valid values are REWORK, TRIVIAL_REBASE, MERGE_FIRST_PARENT_UPDATE, NO_CODE_CHANGE, and NO_CHANGE. +patchset_number | The patch set number, or edit if the patch set is an edit. +ref | The Git reference for the patch set. +uploader | The uploader of the patch set as an AccountInfo entity. + + + +## gerrit_api_obj + +Gerrit API endpoint implementation for feedback migrations and after migration hooks. + + +#### Fields: + +Name | Description +---- | ----------- +url | Return the URL of this endpoint. + + +### gerrit_api_obj.get_change + +Retrieve a Gerrit change. + +`gerritapi.ChangeInfo gerrit_api_obj.get_change(id, include_results=['LABELS'])` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +id | `string`

The change id or change number.

+include_results | `sequence of string`

What to include in the response. See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#query-options

+ + +### gerrit_api_obj.list_changes_by_commit + +Get changes from Gerrit based on a query. See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes. + + +`sequence of gerritapi.ChangeInfo gerrit_api_obj.list_changes_by_commit(commit, include_results=[])` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +commit | `string`

The commit sha to list changes by. See https://gerrit-review.googlesource.com/Documentation/user-search.html#_basic_change_search.

+include_results | `sequence of string`

What to include in the response. See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#query-options

+ + +### gerrit_api_obj.post_review + +Post a review to a Gerrit change for a particular revision. The review will be authored by the user running the tool, or the role account if running in the service. + + +`gerritapi.ReviewResult gerrit_api_obj.post_review(change_id, revision_id, review_input)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +change_id | `string`

The Gerrit change id.

+revision_id | `string`

The revision for which the comment will be posted.

+review_input | `SetReviewInput`

The review to post to Gerrit.

+ + + +## git + +Set of functions to define Git origins and destinations. + + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--experiment-checkout-affected-files` | *boolean* | If set, copybara will only checkout affected files at git origin. Note that this is experimental. +`--git-credential-helper-store-file` | *string* | Credentials store file to be used. See https://git-scm.com/docs/git-credential-store +`--git-no-verify` | *boolean* | Pass the '--no-verify' option to git pushes and commits to disable git commit hooks. +`--git-tag-overwrite` | *boolean* | If set, copybara will force update existing git tag +`--nogit-credential-helper-store` | *boolean* | Disable using credentials store. See https://git-scm.com/docs/git-credential-store +`--nogit-prompt` | *boolean* | Disable username/password prompt and fail if no credentials are found. This flag sets the environment variable GIT_TERMINAL_PROMPT which is intended for automated jobs running Git https://git-scm.com/docs/git/2.3.0#git-emGITTERMINALPROMPTem + + +### git.destination + +Creates a commit in a git repository using the transformed worktree.

Given that Copybara doesn't ask for user/password in the console when doing the push to remote repos, you have to use ssh protocol, have the credentials cached or use a credential manager. + +`gitDestination git.destination(url, push='master', tag_name=None, tag_msg=None, fetch=None, partial_fetch=False, integrates=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the URL to push to as well as the URL from which to get the parent commit

+push | `string`

Reference to use for pushing the change, for example 'master'

+tag_name | `string`

A template string that refers to a tag name. If tag_name exists, overwrite this tag only if flag git-tag-overwrite is set. Note that tag creation is best-effort and migration will succeed even if the tag cannot be created. Usage: Users can use a string or a string with a label. For instance ${label}_tag_name. And the value of label must be in changes' label list. Otherwise, tag won't be created.

+tag_msg | `string`

A template string that refers to the commit msg of a tag. If set, we will create an annotated tag when tag_name is set. Usage: Users can use a string or a string with a label. For instance ${label}_message. And the value of label must be in changes' label list. Otherwise, tag will be created with sha1's commit msg.

+fetch | `string`

Indicates the ref from which to get the parent commit. Defaults to push value if None

+partial_fetch | `boolean`

Please DO NOT set it to True. This feature is not ready.

+integrates | `sequence of git_integrate`

Integrate changes from a url present in the migrated change label. Defaults to a semi-fake merge if COPYBARA_INTEGRATE_REVIEW label is present in the message

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--git-committer-email` | *string* | If set, overrides the committer e-mail for the generated commits in git destination. +`--git-committer-name` | *string* | If set, overrides the committer name for the generated commits in git destination. +`--git-destination-fetch` | *string* | If set, overrides the git destination fetch reference. +`--git-destination-ignore-integration-errors` | *boolean* | If an integration error occurs, ignore it and continue without the integrate +`--git-destination-last-rev-first-parent` | *boolean* | Use git --first-parent flag when looking for last-rev in previous commits +`--git-destination-non-fast-forward` | *boolean* | Allow non-fast-forward pushes to the destination. We only allow this when used with different push != fetch references. +`--git-destination-path` | *string* | If set, the tool will use this directory for the local repository. Note that if the directory exists it needs to be a git repository. Copybara will revert any staged/unstaged changes. +`--git-destination-push` | *string* | If set, overrides the git destination push reference. +`--git-destination-url` | *string* | If set, overrides the git destination URL. +`--nogit-destination-rebase` | *boolean* | Don't rebase the change automatically for workflows CHANGE_REQUEST mode + + +### git.gerrit_api + +Defines a feedback API endpoint for Gerrit, that exposes relevant Gerrit API operations. + +`endpoint_provider of gerrit_api_obj git.gerrit_api(url, checker=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the Gerrit repo URL.

+checker | `checker`

A checker for the Gerrit API transport.

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--gerrit-change-id` | *string* | ChangeId to use in the generated commit message. Use this flag if you want to reuse the same Gerrit review for an export. +`--gerrit-new-change` | *boolean* | Create a new change instead of trying to reuse an existing one. +`--gerrit-topic` | *string* | Gerrit topic to use + + +### git.gerrit_destination + +Creates a change in Gerrit using the transformed worktree. If this is used in iterative mode, then each commit pushed in a single Copybara invocation will have the correct commit parent. The reviews generated can then be easily done in the correct order without rebasing. + +`gerritDestination git.gerrit_destination(url, fetch, push_to_refs_for=fetch value, submit=False, partial_fetch=False, notify=None, change_id_policy='FAIL_IF_PRESENT', allow_empty_diff_patchset=True, reviewers=[], cc=[], labels=[], api_checker=None, integrates=None, topic=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the URL to push to as well as the URL from which to get the parent commit

+fetch | `string`

Indicates the ref from which to get the parent commit

+push_to_refs_for | `string`

Review branch to push the change to, for example setting this to 'feature_x' causes the destination to push to 'refs/for/feature_x'. It defaults to 'fetch' value.

+submit | `boolean`

If true, skip the push thru Gerrit refs/for/branch and directly push to branch. This is effectively a git.destination that sets a Change-Id

+partial_fetch | `boolean`

Please DO NOT set it to True. This feature is not ready.

+notify | `string`

Type of Gerrit notify option (https://gerrit-review.googlesource.com/Documentation/user-upload.html#notify). Sends notifications by default.

+change_id_policy | `string`

What to do in the presence or absent of Change-Id in message:

  • `'REQUIRE'`: Require that the change_id is present in the message as a valid label
  • `'FAIL_IF_PRESENT'`: Fail if found in message
  • `'REUSE'`: Reuse if present. Otherwise generate a new one
  • `'REPLACE'`: Replace with a new one if found

+allow_empty_diff_patchset | `boolean`

By default Copybara will upload a new PatchSet to Gerrit without checking the previous one. If this set to false, Copybara will download current PatchSet and check the diff against the new diff.

+reviewers | `sequence`

The list of the reviewers will be added to gerrit change reviewer listThe element in the list is: an email, for example: "foo@example.com" or label for example: ${SOME_GERRIT_REVIEWER}. These are under the condition of assuming that users have registered to gerrit repos

+cc | `sequence`

The list of the email addresses or users that will be CCed in the review. Can use labels as the `reviewers` field.

+labels | `sequence`

The list of labels to be pushed with the change. The format is the label along with the associated value. For example: Run-Presubmit+1

+api_checker | `checker`

A checker for the Gerrit API endpoint provided for after_migration hooks. This field is not required if the workflow hooks don't use the origin/destination endpoints.

+integrates | `sequence of git_integrate`

Integrate changes from a url present in the migrated change label. Defaults to a semi-fake merge if COPYBARA_INTEGRATE_REVIEW label is present in the message

+topic | `string`

Sets the topic of the Gerrit change created.

By default it sets no topic. This field accepts a template with labels. For example: `"topic_${CONTEXT_REFERENCE}"`

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--git-committer-email` | *string* | If set, overrides the committer e-mail for the generated commits in git destination. +`--git-committer-name` | *string* | If set, overrides the committer name for the generated commits in git destination. +`--git-destination-fetch` | *string* | If set, overrides the git destination fetch reference. +`--git-destination-ignore-integration-errors` | *boolean* | If an integration error occurs, ignore it and continue without the integrate +`--git-destination-last-rev-first-parent` | *boolean* | Use git --first-parent flag when looking for last-rev in previous commits +`--git-destination-non-fast-forward` | *boolean* | Allow non-fast-forward pushes to the destination. We only allow this when used with different push != fetch references. +`--git-destination-path` | *string* | If set, the tool will use this directory for the local repository. Note that if the directory exists it needs to be a git repository. Copybara will revert any staged/unstaged changes. +`--git-destination-push` | *string* | If set, overrides the git destination push reference. +`--git-destination-url` | *string* | If set, overrides the git destination URL. +`--nogit-destination-rebase` | *boolean* | Don't rebase the change automatically for workflows CHANGE_REQUEST mode + + +### git.gerrit_origin + +Defines a Git origin for Gerrit reviews. + +Implicit labels that can be used/exposed: + + - GERRIT_CHANGE_NUMBER: The change number for the Gerrit review. + - GERRIT_CHANGE_ID: The change id for the Gerrit review. + - GERRIT_CHANGE_DESCRIPTION: The description of the Gerrit review. + - COPYBARA_INTEGRATE_REVIEW: A label that when exposed, can be used to integrate automatically in the reverse workflow. + - GERRIT_CHANGE_BRANCH: The destination branch for thechange + - GERRIT_CHANGE_TOPIC: The change topic + - GERRIT_COMPLETE_CHANGE_ID: Complete Change-Id with project, branch and Change-Id + - GERRIT_OWNER_EMAIL: Owner email + - GERRIT_REVIEWER_EMAIL: Multiple value field with the email of the reviewers + - GERRIT_CC_EMAIL: Multiple value field with the email of the people/groups in cc + + +`gitOrigin git.gerrit_origin(url, ref=None, submodules='NO', first_parent=True, partial_fetch=False, api_checker=None, patch=None, branch=None, describe_version=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the URL of the git repository

+ref | `string`

DEPRECATED. Use git.origin for submitted branches.

+submodules | `string`

Download submodules. Valid values: NO, YES, RECURSIVE.

+first_parent | `boolean`

If true, it only uses the first parent when looking for changes. Note that when disabled in ITERATIVE mode, it will try to do a migration for each change of the merged branch.

+partial_fetch | `boolean`

Please DO NOT set it to True. This feature is not ready.

+api_checker | `checker`

A checker for the Gerrit API endpoint provided for after_migration hooks. This field is not required if the workflow hooks don't use the origin/destination endpoints.

+patch | `transformation`

Patch the checkout dir. The difference with `patch.apply` transformation is that here we can apply it using three-way

+branch | `string`

Limit the import to changes that are for this branch. By default imports everything.

+describe_version | `boolean`

Download tags and use 'git describe' to create two labels with a meaningful version:

- `GIT_DESCRIBE_CHANGE_VERSION`: The version for the change or changes being migrated. The value changes per change in `ITERATIVE` mode and will be the latest migrated change in `SQUASH` (In other words, doesn't include excluded changes). this is normally what users want to use.
- `GIT_DESCRIBE_REQUESTED_VERSION`: `git describe` for the requested/head version. Constant in `ITERATIVE` mode and includes filtered changes.

+ + +### git.gerrit_trigger + +Defines a feedback trigger based on updates on a Gerrit change. + +`gerritTrigger git.gerrit_trigger(url, checker=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the Gerrit repo URL.

+checker | `checker`

A checker for the Gerrit API transport provided by this trigger.

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--gerrit-change-id` | *string* | ChangeId to use in the generated commit message. Use this flag if you want to reuse the same Gerrit review for an export. +`--gerrit-new-change` | *boolean* | Create a new change instead of trying to reuse an existing one. +`--gerrit-topic` | *string* | Gerrit topic to use + + +### git.github_api + +Defines a feedback API endpoint for GitHub, that exposes relevant GitHub API operations. + +`endpoint_provider of github_api_obj git.github_api(url, checker=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the GitHub repo URL.

+checker | `checker`

A checker for the GitHub API transport.

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field + + +### git.github_destination + +Creates a commit in a GitHub repository branch (for example master). For creating PullRequest use git.github_pr_destination. + +`gitDestination git.github_destination(url, push='master', fetch=None, pr_branch_to_update=None, partial_fetch=False, delete_pr_branch=False, integrates=None, api_checker=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the URL to push to as well as the URL from which to get the parent commit

+push | `string`

Reference to use for pushing the change, for example 'master'

+fetch | `string`

Indicates the ref from which to get the parent commit. Defaults to push value if None

+pr_branch_to_update | `string`

A template string that refers to a pull request branch in the same repository will be updated to current commit of this push branch only if pr_branch_to_update exists. The reason behind this field is that presubmiting changes creates and leaves a pull request open. By using this, we can automerge/close this type of pull requests. As a result, users will see this pr_branch_to_update as merged to this push branch. Usage: Users can use a string or a string with a label. For instance ${label}_pr_branch_name. And the value of label must be in changes' label list. Otherwise, nothing will happen.

+partial_fetch | `boolean`

Please DO NOT set it to True. This feature is not ready.

+delete_pr_branch | `boolean`

When `pr_branch_to_update` is enabled, it will delete the branch reference after the push to the branch and main branch (i.e master) happens. This allows to cleanup temporary branches created for testing.

+integrates | `sequence of git_integrate`

Integrate changes from a url present in the migrated change label. Defaults to a semi-fake merge if COPYBARA_INTEGRATE_REVIEW label is present in the message

+api_checker | `checker`

A checker for the Gerrit API endpoint provided for after_migration hooks. This field is not required if the workflow hooks don't use the origin/destination endpoints.

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--git-committer-email` | *string* | If set, overrides the committer e-mail for the generated commits in git destination. +`--git-committer-name` | *string* | If set, overrides the committer name for the generated commits in git destination. +`--git-destination-fetch` | *string* | If set, overrides the git destination fetch reference. +`--git-destination-ignore-integration-errors` | *boolean* | If an integration error occurs, ignore it and continue without the integrate +`--git-destination-last-rev-first-parent` | *boolean* | Use git --first-parent flag when looking for last-rev in previous commits +`--git-destination-non-fast-forward` | *boolean* | Allow non-fast-forward pushes to the destination. We only allow this when used with different push != fetch references. +`--git-destination-path` | *string* | If set, the tool will use this directory for the local repository. Note that if the directory exists it needs to be a git repository. Copybara will revert any staged/unstaged changes. +`--git-destination-push` | *string* | If set, overrides the git destination push reference. +`--git-destination-url` | *string* | If set, overrides the git destination URL. +`--nogit-destination-rebase` | *boolean* | Don't rebase the change automatically for workflows CHANGE_REQUEST mode + + +### git.github_origin + +Defines a Git origin for a Github repository. This origin should be used for public branches. Use github_pr_origin for importing Pull Requests. + +`gitOrigin git.github_origin(url, ref=None, submodules='NO', first_parent=True, partial_fetch=False, patch=None, describe_version=None, version_selector=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the URL of the git repository

+ref | `string`

Represents the default reference that will be used for reading the revision from the git repository. For example: 'master'

+submodules | `string`

Download submodules. Valid values: NO, YES, RECURSIVE.

+first_parent | `boolean`

If true, it only uses the first parent when looking for changes. Note that when disabled in ITERATIVE mode, it will try to do a migration for each change of the merged branch.

+partial_fetch | `boolean`

Please DO NOT set it to True. This feature is not ready.

+patch | `transformation`

Patch the checkout dir. The difference with `patch.apply` transformation is that here we can apply it using three-way

+describe_version | `boolean`

Download tags and use 'git describe' to create two labels with a meaningful version:

- `GIT_DESCRIBE_CHANGE_VERSION`: The version for the change or changes being migrated. The value changes per change in `ITERATIVE` mode and will be the latest migrated change in `SQUASH` (In other words, doesn't include excluded changes). this is normally what users want to use.
- `GIT_DESCRIBE_REQUESTED_VERSION`: `git describe` for the requested/head version. Constant in `ITERATIVE` mode and includes filtered changes.

+version_selector | `latestVersionSelector`

Select a custom version (tag)to migrate instead of 'ref'

+ + +### git.github_pr_destination + +Creates changes in a new pull request in the destination. + +`gitHubPrDestination git.github_pr_destination(url, destination_ref="master", pr_branch=None, partial_fetch=False, title=None, body=None, integrates=None, api_checker=None, update_description=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Url of the GitHub project. For example "https://github.com/google/copybara'"

+destination_ref | `string`

Destination reference for the change. By default 'master'

+pr_branch | `string`

Customize the pull request branch. Any variable present in the message in the form of ${CONTEXT_REFERENCE} will be replaced by the corresponding stable reference (head, PR number, Gerrit change number, etc.).

+partial_fetch | `boolean`

Please DO NOT set it to True. This feature is not ready.

+title | `string`

When creating (or updating if `update_description` is set) a pull request, use this title. By default it uses the change first line. This field accepts a template with labels. For example: `"Change ${CONTEXT_REFERENCE}"`

+body | `string`

When creating (or updating if `update_description` is set) a pull request, use this body. By default it uses the change summary. This field accepts a template with labels. For example: `"Change ${CONTEXT_REFERENCE}"`

+integrates | `sequence of git_integrate`

Integrate changes from a url present in the migrated change label. Defaults to a semi-fake merge if COPYBARA_INTEGRATE_REVIEW label is present in the message

+api_checker | `checker`

A checker for the GitHub API endpoint provided for after_migration hooks. This field is not required if the workflow hooks don't use the origin/destination endpoints.

+update_description | `boolean`

By default, Copybara only set the title and body of the PR when creating the PR. If this field is set to true, it will update those fields for every update.

+ + +#### Examples: + + +##### Common usage: + +Create a branch by using copybara's computerIdentity algorithm: + +```python +git.github_pr_destination( + url = "https://github.com/google/copybara", + destination_ref = "master", + ) +``` + + +##### Using pr_branch with label: + +Customize pr_branch with context reference: + +```python +git.github_pr_destination( + url = "https://github.com/google/copybara", + destination_ref = "master", + pr_branch = 'test_${CONTEXT_REFERENCE}', + ) +``` + + +##### Using pr_branch with constant string: + +Customize pr_branch with a constant string: + +```python +git.github_pr_destination( + url = "https://github.com/google/copybara", + destination_ref = "master", + pr_branch = 'test_my_branch', + ) +``` + + + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--git-committer-email` | *string* | If set, overrides the committer e-mail for the generated commits in git destination. +`--git-committer-name` | *string* | If set, overrides the committer name for the generated commits in git destination. +`--git-destination-fetch` | *string* | If set, overrides the git destination fetch reference. +`--git-destination-ignore-integration-errors` | *boolean* | If an integration error occurs, ignore it and continue without the integrate +`--git-destination-last-rev-first-parent` | *boolean* | Use git --first-parent flag when looking for last-rev in previous commits +`--git-destination-non-fast-forward` | *boolean* | Allow non-fast-forward pushes to the destination. We only allow this when used with different push != fetch references. +`--git-destination-path` | *string* | If set, the tool will use this directory for the local repository. Note that if the directory exists it needs to be a git repository. Copybara will revert any staged/unstaged changes. +`--git-destination-push` | *string* | If set, overrides the git destination push reference. +`--git-destination-url` | *string* | If set, overrides the git destination URL. +`--github-destination-pr-branch` | *string* | If set, uses this branch for creating the pull request instead of using a generated one +`--github-destination-pr-create` | *boolean* | If the pull request should be created +`--nogit-destination-rebase` | *boolean* | Don't rebase the change automatically for workflows CHANGE_REQUEST mode + + +### git.github_pr_origin + +Defines a Git origin for Github pull requests. + +Implicit labels that can be used/exposed: + + - GITHUB_PR_NUMBER: The pull request number if the reference passed was in the form of `https://github.com/project/pull/123`, `refs/pull/123/head` or `refs/pull/123/master`. + - COPYBARA_INTEGRATE_REVIEW: A label that when exposed, can be used to integrate automatically in the reverse workflow. + - GITHUB_BASE_BRANCH: The base branch name used for the Pull Request. + - GITHUB_BASE_BRANCH_SHA1: The base branch SHA-1 used as baseline. + - GITHUB_PR_TITLE: Title of the Pull Request. + - GITHUB_PR_BODY: Body of the Pull Request. + - GITHUB_PR_URL: GitHub url of the Pull Request. + - GITHUB_PR_HEAD_SHA: The SHA-1 of the head commit of the pull request. + - GITHUB_PR_USER: The login of the author the pull request. + - GITHUB_PR_ASSIGNEE: A repeated label with the login of the assigned users. + - GITHUB_PR_REVIEWER_APPROVER: A repeated label with the login of users that have participated in the review and that can approve the import. Only populated if `review_state` field is set. Every reviewers type matching `review_approvers` will be added to this list. + - GITHUB_PR_REVIEWER_OTHER: A repeated label with the login of users that have participated in the review but cannot approve the import. Only populated if `review_state` field is set. + + +`gitHubPROrigin git.github_pr_origin(url, use_merge=False, required_labels=[], retryable_labels=[], submodules='NO', baseline_from_branch=False, first_parent=True, partial_fetch=False, state='OPEN', review_state=None, review_approvers=["COLLABORATOR", "MEMBER", "OWNER"], api_checker=None, patch=None, branch=None, describe_version=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the URL of the GitHub repository

+use_merge | `boolean`

If the content for refs/pull//merge should be used instead of the PR head. The GitOrigin-RevId still will be the one from refs/pull//head revision.

+required_labels | `sequence of string`

Required labels to import the PR. All the labels need to be present in order to migrate the Pull Request.

+retryable_labels | `sequence of string`

Required labels to import the PR that should be retried. This parameter must be a subset of required_labels.

+submodules | `string`

Download submodules. Valid values: NO, YES, RECURSIVE.

+baseline_from_branch | `boolean`

WARNING: Use this field only for github -> git CHANGE_REQUEST workflows.
When the field is set to true for CHANGE_REQUEST workflows it will find the baseline comparing the Pull Request with the base branch instead of looking for the *-RevId label in the commit message.

+first_parent | `boolean`

If true, it only uses the first parent when looking for changes. Note that when disabled in ITERATIVE mode, it will try to do a migration for each change of the merged branch.

+partial_fetch | `boolean`

Please DO NOT set it to True. This feature is not ready.

+state | `string`

Only migrate Pull Request with that state. Possible values: `'OPEN'`, `'CLOSED'` or `'ALL'`. Default 'OPEN'

+review_state | `string`

Required state of the reviews associated with the Pull Request Possible values: `'HEAD_COMMIT_APPROVED'`, `'ANY_COMMIT_APPROVED'`, `'HAS_REVIEWERS'` or `'ANY'`. Default `None`. This field is required if the user wants `GITHUB_PR_REVIEWER_APPROVER` and `GITHUB_PR_REVIEWER_OTHER` labels populated

+review_approvers | `sequence of string`

The set of reviewer types that are considered for approvals. In order to have any effect, `review_state` needs to be set. GITHUB_PR_REVIEWER_APPROVER` will be populated for these types. See the valid types here: https://developer.github.com/v4/enum/commentauthorassociation/

+api_checker | `checker`

A checker for the GitHub API endpoint provided for after_migration hooks. This field is not required if the workflow hooks don't use the origin/destination endpoints.

+patch | `transformation`

Patch the checkout dir. The difference with `patch.apply` transformation is that here we can apply it using three-way

+branch | `string`

If set, it will only migrate pull requests for this base branch

+describe_version | `boolean`

Download tags and use 'git describe' to create two labels with a meaningful version:

- `GIT_DESCRIBE_CHANGE_VERSION`: The version for the change or changes being migrated. The value changes per change in `ITERATIVE` mode and will be the latest migrated change in `SQUASH` (In other words, doesn't include excluded changes). this is normally what users want to use.
- `GIT_DESCRIBE_REQUESTED_VERSION`: `git describe` for the requested/head version. Constant in `ITERATIVE` mode and includes filtered changes.

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--github-force-import` | *boolean* | Force import regardless of the state of the PR +`--github-pr-merge` | *boolean* | Override merge bit from config +`--github-required-label` | *list<string>* | Required labels in the Pull Request to be imported by github_pr_origin +`--github-retryable-label` | *list<string>* | Required labels in the Pull Request that should be retryed to be imported by github_pr_origin +`--github-skip-required-labels` | *boolean* | Skip checking labels for importing Pull Requests. Note that this is dangerous as it might import an unsafe PR. + + +### git.github_trigger + +Defines a feedback trigger based on updates on a GitHub PR. + +`gitHubTrigger git.github_trigger(url, checker=None, events=[])` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the GitHub repo URL.

+checker | `checker`

A checker for the GitHub API transport provided by this trigger.

+events | `sequence of string`

Type of events to subscribe. Valid values are: `'ISSUES'`, `'ISSUE_COMMENT'`, `'PULL_REQUEST'`, `'PULL_REQUEST_REVIEW_COMMENT'`, `'PUSH'`, `'STATUS'`,

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field + + +### git.integrate + +Integrate changes from a url present in the migrated change label. + +`git_integrate git.integrate(label="COPYBARA_INTEGRATE_REVIEW", strategy="FAKE_MERGE_AND_INCLUDE_FILES", ignore_errors=True)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +label | `string`

The migration label that will contain the url to the change to integrate.

+strategy | `string`

How to integrate the change:

  • 'FAKE_MERGE': Add the url revision/reference as parent of the migration change but ignore all the files from the url. The commit message will be a standard merge one but will include the corresponding RevId label
  • 'FAKE_MERGE_AND_INCLUDE_FILES': Same as 'FAKE_MERGE' but any change to files that doesn't match destination_files will be included as part of the merge commit. So it will be a semi fake merge: Fake for destination_files but merge for non destination files.
  • 'INCLUDE_FILES': Same as 'FAKE_MERGE_AND_INCLUDE_FILES' but it it doesn't create a merge but only include changes not matching destination_files

+ignore_errors | `boolean`

If we should ignore integrate errors and continue the migration without the integrate

+ + +#### Example: + + +##### Integrate changes from a review url: + +Assuming we have a git.destination defined like this: + +```python +git.destination( + url = "https://example.com/some_git_repo", + integrates = [git.integrate()], + +) +``` + +It will look for `COPYBARA_INTEGRATE_REVIEW` label during the worklow migration. If the label is found, it will fetch the git url and add that change as an additional parent to the migration commit (merge). It will fake-merge any change from the url that matches destination_files but it will include changes not matching it. + + + +### git.latest_version + +Customize what version of the available branches and tags to pick. By default it ignores the reference passed as parameter. Using `force:reference` in the CLI will force to use that reference instead. + +`latestVersionSelector git.latest_version(refspec_format="refs/tags/${n0}.${n1}.${n2}", refspec_groups={'n0' : '[0-9]+', 'n1' : '[0-9]+', 'n2' : '[0-9]+'})` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +refspec_format | `string`

The format of the branch/tag

+refspec_groups | `dict`

A set of named regexes that can be used to match part of the versions.Copybara uses [re2](https://github.com/google/re2/wiki/Syntax) syntax. Use the following nomenclature n0, n1, n2 for the version part (will use numeric sorting) or s0, s1, s2 (alphabetic sorting). Note that there can be mixed but the numbers cannot be repeated. In other words n0, s1, n2 is valid but not n0, s0, n1. n0 has more priority than n1. If there are fields where order is not important, use s(N+1) where N ist he latest sorted field. Example {"n0": "[0-9]+", "s1": "[a-z]+"}

+ + +### git.mirror + +Mirror git references between repositories + +`git.mirror(name, origin, destination, refspecs=['refs/heads/*'], prune=False, partial_fetch=False, description=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +name | `string`

Migration name

+origin | `string`

Indicates the URL of the origin git repository

+destination | `string`

Indicates the URL of the destination git repository

+refspecs | `sequence of string`

Represents a list of git refspecs to mirror between origin and destination. For example 'refs/heads/*:refs/remotes/origin/*' will mirror any reference inside refs/heads to refs/remotes/origin.

+prune | `boolean`

Remove remote refs that don't have a origin counterpart

+partial_fetch | `boolean`

Please DO NOT set it to True. This feature is not ready.

+description | `string`

A description of what this workflow achieves

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--git-mirror-force` | *boolean* | Force push even if it is not fast-forward + + +### git.origin + +Defines a standard Git origin. For Git specific origins use: `github_origin` or `gerrit_origin`.

All the origins in this module accept several string formats as reference (When copybara is called in the form of `copybara config workflow reference`):
  • **Branch name:** For example `master`
  • **An arbitrary reference:** `refs/changes/20/50820/1`
  • **A SHA-1:** Note that it has to be reachable from the default refspec
  • **A Git repository URL and reference:** `http://github.com/foo master`
  • **A GitHub pull request URL:** `https://github.com/some_project/pull/1784`

So for example, Copybara can be invoked for a `git.origin` in the CLI as:
`copybara copy.bara.sky my_workflow https://github.com/some_project/pull/1784`
This will use the pull request as the origin URL and reference. + +`gitOrigin git.origin(url, ref=None, submodules='NO', include_branch_commit_logs=False, first_parent=True, partial_fetch=False, patch=None, describe_version=None, version_selector=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the URL of the git repository

+ref | `string`

Represents the default reference that will be used for reading the revision from the git repository. For example: 'master'

+submodules | `string`

Download submodules. Valid values: NO, YES, RECURSIVE.

+include_branch_commit_logs | `boolean`

Whether to include raw logs of branch commits in the migrated change message.WARNING: This field is deprecated in favor of 'first_parent' one. This setting *only* affects merge commits.

+first_parent | `boolean`

If true, it only uses the first parent when looking for changes. Note that when disabled in ITERATIVE mode, it will try to do a migration for each change of the merged branch.

+partial_fetch | `boolean`

Please DO NOT set it to True. This feature is not ready.

+patch | `transformation`

Patch the checkout dir. The difference with `patch.apply` transformation is that here we can apply it using three-way

+describe_version | `boolean`

Download tags and use 'git describe' to create two labels with a meaningful version:

- `GIT_DESCRIBE_CHANGE_VERSION`: The version for the change or changes being migrated. The value changes per change in `ITERATIVE` mode and will be the latest migrated change in `SQUASH` (In other words, doesn't include excluded changes). this is normally what users want to use.
- `GIT_DESCRIBE_REQUESTED_VERSION`: `git describe` for the requested/head version. Constant in `ITERATIVE` mode and includes filtered changes.

+version_selector | `latestVersionSelector`

Select a custom version (tag)to migrate instead of 'ref'

+ + +### git.review_input + +Creates a review to be posted on Gerrit. + +`SetReviewInput git.review_input(labels={}, message=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +labels | `dict`

The labels to post.

+message | `string`

The message to be added as review comment.

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--gerrit-change-id` | *string* | ChangeId to use in the generated commit message. Use this flag if you want to reuse the same Gerrit review for an export. +`--gerrit-new-change` | *boolean* | Create a new change instead of trying to reuse an existing one. +`--gerrit-topic` | *string* | Gerrit topic to use + + + +## github_api_obj + +GitHub API endpoint implementation for feedback migrations and after migration hooks. + + +#### Fields: + +Name | Description +---- | ----------- +url | Return the URL of this endpoint. + + +### github_api_obj.add_label + +Add labels to a PR/issue + +`github_api_obj.add_label(number, labels)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +number | `integer`

Pull Request number

+labels | `sequence of string`

List of labels to add.

+ + +### github_api_obj.create_status + +Create or update a status for a commit. Returns the status created. + +`github_api_status_obj github_api_obj.create_status(sha, state, context, description, target_url=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +sha | `string`

The SHA-1 for which we want to create or update the status

+state | `string`

The state of the commit status: 'success', 'error', 'pending' or 'failure'

+context | `string`

The context for the commit status. Use a value like 'copybara/import_successful' or similar

+description | `string`

Description about what happened

+target_url | `string`

Url with expanded information about the event

+ + +### github_api_obj.delete_reference + +Delete a reference. + +`github_api_obj.delete_reference(ref)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +ref | `string`

The name of the reference.

+ + +### github_api_obj.get_authenticated_user + +Get autenticated user info, return null if not found + +`github_api_user_obj github_api_obj.get_authenticated_user()` + + +### github_api_obj.get_check_runs + +Get the list of check runs for a sha. https://developer.github.com/v3/checks/runs/#check-runs + +`github_check_runs_obj github_api_obj.get_check_runs(sha)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +sha | `string`

The SHA-1 for which we want to get the check runs

+ + +### github_api_obj.get_combined_status + +Get the combined status for a commit. Returns None if not found. + +`github_api_combined_status_obj github_api_obj.get_combined_status(ref)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +ref | `string`

The SHA-1 or ref for which we want to get the combined status

+ + +### github_api_obj.get_commit + +Get information for a commit in GitHub. Returns None if not found. + +`github_api_github_commit_obj github_api_obj.get_commit(ref)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +ref | `string`

The SHA-1 for which we want to get the combined status

+ + +### github_api_obj.get_pull_request_comment + +Get a pull request comment + +`github_api_pull_request_comment_obj github_api_obj.get_pull_request_comment(comment_id)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +comment_id | `string`

Comment identifier

+ + +### github_api_obj.get_pull_request_comments + +Get all pull request comments + +`sequence of github_api_pull_request_comment_obj github_api_obj.get_pull_request_comments(number)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +number | `integer`

Pull Request number

+ + +### github_api_obj.get_pull_requests + +Get Pull Requests for a repo + +`immutableList<e> github_api_obj.get_pull_requests(head_prefix=None, base_prefix=None, state="OPEN", sort="CREATED", direction="ASC")` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +head_prefix | `string`

Only return PRs wher the branch name has head_prefix

+base_prefix | `string`

Only return PRs where the destination branch name has base_prefix

+state | `string`

State of the Pull Request. Can be `"OPEN"`, `"CLOSED"` or `"ALL"`

+sort | `string`

Sort filter for retrieving the Pull Requests. Can be `"CREATED"`, `"UPDATED"` or `"POPULARITY"`

+direction | `string`

Direction of the filter. Can be `"ASC"` or `"DESC"`

+ + +### github_api_obj.get_reference + +Get a reference SHA-1 from GitHub. Returns None if not found. + +`github_api_ref_obj github_api_obj.get_reference(ref)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +ref | `string`

The name of the reference. For example: "refs/heads/branchName".

+ + +### github_api_obj.get_references + +Get all the reference SHA-1s from GitHub. Note that Copybara only returns a maximum number of 500. + +`sequence of github_api_ref_obj github_api_obj.get_references()` + + +### github_api_obj.update_pull_request + +Update Pull Requests for a repo. Returns None if not found + +`github_api_pull_request_obj github_api_obj.update_pull_request(number, title=None, body=None, state=None)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +number | `integer`

Pull Request number

+title | `string`

New Pull Request title

+body | `string`

New Pull Request body

+state | `string`

State of the Pull Request. Can be `"OPEN"`, `"CLOSED"`

+ + +### github_api_obj.update_reference + +Update a reference to point to a new commit. Returns the info of the reference. + +`github_api_ref_obj github_api_obj.update_reference(ref, sha, force)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +ref | `string`

The name of the reference.

+sha | `string`

The id for the commit status.

+force | `boolean`

Indicates whether to force the update or to make sure the update is a fast-forward update. Leaving this out or setting it to false will make sure you're not overwriting work. Default: false

+ + + +## Globals + +Global functions available in Copybara + + +### glob + +Glob returns a list of every file in the workdir that matches at least one pattern in include and does not match any of the patterns in exclude. + +`glob glob(include, exclude=[])` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +include | `sequence of string`

The list of glob patterns to include

+exclude | `sequence of string`

The list of glob patterns to exclude

+ + +#### Examples: + + +##### Simple usage: + +Include all the files under a folder except for `internal` folder files: + +```python +glob(["foo/**"], exclude = ["foo/internal/**"]) +``` + + +##### Multiple folders: + +Globs can have multiple inclusive rules: + +```python +glob(["foo/**", "bar/**", "baz/**.java"]) +``` + +This will include all files inside `foo` and `bar` folders and Java files inside `baz` folder. + + +##### Multiple excludes: + +Globs can have multiple exclusive rules: + +```python +glob(["foo/**"], exclude = ["foo/internal/**", "foo/confidential/**" ]) +``` + +Include all the files of `foo` except the ones in `internal` and `confidential` folders + + +##### All BUILD files recursively: + +Copybara uses Java globbing. The globbing is very similar to Bash one. This means that recursive globbing for a filename is a bit more tricky: + +```python +glob(["BUILD", "**/BUILD"]) +``` + +This is the correct way of matching all `BUILD` files recursively, including the one in the root. `**/BUILD` would only match `BUILD` files in subdirectories. + + +##### Matching multiple strings with one expression: + +While two globs can be used for matching two directories, there is a more compact approach: + +```python +glob(["{java,javatests}/**"]) +``` + +This matches any file in `java` and `javatests` folders. + + +##### Glob union: + +This is useful when you want to exclude a broad subset of files but you want to still include some of those files. + +```python +glob(["folder/**"], exclude = ["folder/**.excluded"]) + glob(['folder/includeme.excluded']) +``` + +This matches all the files in `folder`, excludes all files in that folder that ends with `.excluded` but keeps `folder/includeme.excluded`

`+` operator for globs is equivalent to `OR` operation. + + + +### new_author + +Create a new author from a string with the form 'name ' + +`author new_author(author_string)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +author_string | `string`

A string representation of the author with the form 'name '

+ + +#### Example: + + +##### Create a new author: + + + +```python +new_author('Foo Bar ') +``` + + + +### parse_message + +Returns a ChangeMessage parsed from a well formed string. + +`ChangeMessage parse_message(message)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +message | `string`

The contents of the change message

+ + + +## hg + +Set of functions to define Mercurial (Hg) origins and destinations. + + +### hg.origin + +EXPERIMENTAL: Defines a standard Mercurial (Hg) origin. + +`hgOrigin hg.origin(url, ref="default")` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +url | `string`

Indicates the URL of the Hg repository

+ref | `string`

Represents the default reference that will be used to read a revision from the repository. The reference defaults to `default`, the most recent revision on the default branch. References can be in a variety of formats:

  • A global identifier for a revision. Example: f4e0e692208520203de05557244e573e981f6c72
  • A bookmark in the repository.
  • A branch in the repository, which returns the tip of that branch. Example: default
  • A tag in the repository. Example: tip

+ + + +## mapping_function + +A function that given an object can map to another object + + + +## metadata + +Core transformations for the change metadata + + +### metadata.add_header + +Adds a header line to the commit message. Any variable present in the message in the form of ${LABEL_NAME} will be replaced by the corresponding label in the message. Note that this requires that the label is already in the message or in any of the changes being imported. The label in the message takes priority over the ones in the list of original messages of changes imported. + + +`transformation metadata.add_header(text, ignore_label_not_found=False, new_line=True)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +text | `string`

The header text to include in the message. For example '[Import of foo ${LABEL}]'. This would construct a message resolving ${LABEL} to the corresponding label.

+ignore_label_not_found | `boolean`

If a label used in the template is not found, ignore the error and don't add the header. By default it will stop the migration and fail.

+new_line | `boolean`

If a new line should be added between the header and the original message. This allows to create messages like `HEADER: ORIGINAL_MESSAGE`

+ + +#### Examples: + + +##### Add a header always: + +Adds a header to any message + +```python +metadata.add_header("COPYBARA CHANGE") +``` + +Messages like: + +``` +A change + +Example description for +documentation +``` + +Will be transformed into: + +``` +COPYBARA CHANGE +A change + +Example description for +documentation +``` + + + + +##### Add a header that uses a label: + +Adds a header to messages that contain a label. Otherwise it skips the message manipulation. + +```python +metadata.add_header("COPYBARA CHANGE FOR https://github.com/myproject/foo/pull/${GITHUB_PR_NUMBER}", + ignore_label_not_found = True, +) +``` + +A change message, imported using git.github_pr_origin, like: + +``` +A change + +Example description for +documentation +``` + +Will be transformed into: + +``` +COPYBARA CHANGE FOR https://github.com/myproject/foo/pull/1234 +Example description for +documentation +``` + +Assuming the PR number is 1234. But any change without that label will not be transformed. + + +##### Add a header without new line: + +Adds a header without adding a new line before the original message: + +```python +metadata.add_header("COPYBARA CHANGE: ", new_line = False) +``` + +Messages like: + +``` +A change + +Example description for +documentation +``` + +Will be transformed into: + +``` +COPYBARA CHANGE: A change + +Example description for +documentation +``` + + + + + +### metadata.expose_label + +Certain labels are present in the internal metadata but are not exposed in the message by default. This transformations find a label in the internal metadata and exposes it in the message. If the label is already present in the message it will update it to use the new name and separator. + +`transformation metadata.expose_label(name, new_name=label, separator="=", ignore_label_not_found=True, all=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +name | `string`

The label to search

+new_name | `string`

The name to use in the message

+separator | `string`

The separator to use when adding the label to the message

+ignore_label_not_found | `boolean`

If a label is not found, ignore the error and continue.

+all | `boolean`

By default Copybara tries to find the most relevant instance of the label. First looking into the message and then looking into the changes in order. If this field is true it exposes all the matches instead.

+ + +#### Examples: + + +##### Simple usage: + +Expose a hidden label called 'REVIEW_URL': + +```python +metadata.expose_label('REVIEW_URL') +``` + +This would add it as `REVIEW_URL=the_value`. + + +##### New label name: + +Expose a hidden label called 'REVIEW_URL' as GIT_REVIEW_URL: + +```python +metadata.expose_label('REVIEW_URL', 'GIT_REVIEW_URL') +``` + +This would add it as `GIT_REVIEW_URL=the_value`. + + +##### Custom separator: + +Expose the label with a custom separator + +```python +metadata.expose_label('REVIEW_URL', separator = ': ') +``` + +This would add it as `REVIEW_URL: the_value`. + + +##### Expose multiple labels: + +Expose all instances of a label in all the changes (SQUASH for example) + +```python +metadata.expose_label('REVIEW_URL', all = True) +``` + +This would add 0 or more `REVIEW_URL: the_value` labels to the message. + + + +### metadata.map_author + +Map the author name and mail to another author. The mapping can be done by both name and mail or only using any of the two. + +`transformation metadata.map_author(authors, reversible=False, noop_reverse=False, fail_if_not_found=False, reverse_fail_if_not_found=False, map_all_changes=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +authors | `dict`

The author mapping. Keys can be in the form of 'Your Name', 'some@mail' or 'Your Name '. The mapping applies heuristics to know which field to use in the mapping. The value has to be always in the form of 'Your Name '

+reversible | `boolean`

If the transform is automatically reversible. Workflows using the reverse of this transform will be able to automatically map values to keys.

+noop_reverse | `boolean`

If true, the reversal of the transformation doesn't do anything. This is useful to avoid having to write `core.transformation(metadata.map_author(...), reversal = [])`.

+fail_if_not_found | `boolean`

Fail if a mapping cannot be found. Helps discovering early authors that should be in the map

+reverse_fail_if_not_found | `boolean`

Same as fail_if_not_found but when the transform is used in a inverse workflow.

+map_all_changes | `boolean`

If all changes being migrated should be mapped. Useful for getting a mapped metadata.squash_notes. By default we only map the current author.

+ + +#### Example: + + +##### Map some names, emails and complete authors: + +Here we show how to map authors using different options: + +```python +metadata.map_author({ + 'john' : 'Some Person ', + 'madeupexample@google.com' : 'Other Person ', + 'John Example ' : 'Another Person ', +}) +``` + + + +### metadata.map_references + +Allows updating links to references in commit messages to match the destination's format. Note that this will only consider the 5000 latest commits. + +`referenceMigrator metadata.map_references(before, after, regex_groups={}, additional_import_labels=[])` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +before | `string`

Template for origin references in the change message. Use a '${reference}' token to capture the actual references. E.g. if the origin uses linkslike 'http://changes?1234', the template would be 'http://internalReviews.com/${reference}', with reference_regex = '[0-9]+'

+after | `string`

Format for references in the destination, use the token '${reference}' to represent the destination reference. E.g. 'http://changes(${reference})'.

+regex_groups | `dict`

Regexes for the ${reference} token's content. Requires one 'before_ref' entry matching the ${reference} token's content on the before side. Optionally accepts one 'after_ref' used for validation. Copybara uses [re2](https://github.com/google/re2/wiki/Syntax) syntax.

+additional_import_labels | `sequence of string`

Meant to be used when migrating from another tool: Per default, copybara will only recognize the labels defined in the workflow's endpoints. The tool will use these additional labels to find labels created by other invocations and tools.

+ + +#### Example: + + +##### Map references, origin source of truth: + +Finds links to commits in change messages, searches destination to find the equivalent reference in destination. Then replaces matches of 'before' with 'after', replacing the subgroup matched with the destination reference. Assume a message like 'Fixes bug introduced in origin/abcdef', where the origin change 'abcdef' was migrated as '123456' to the destination. + +```python +metadata.map_references( + before = "origin/${reference}", + after = "destination/${reference}", + regex_groups = { + "before_ref": "[0-9a-f]+", + "after_ref": "[0-9]+", + }, +) +``` + +This would be translated into 'Fixes bug introduced in destination/123456', provided that a change with the proper label was found - the message remains unchanged otherwise. + + + +### metadata.remove_label + +Remove a label from the message + +`transformation metadata.remove_label(name)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +name | `string`

The label name

+ + +#### Example: + + +##### Remove a label: + +Remove Change-Id label from the message: + +```python +metadata.remove_label('Change-Id') +``` + + + +### metadata.replace_message + +Replace the change message with a template text. Any variable present in the message in the form of ${LABEL_NAME} will be replaced by the corresponding label in the message. Note that this requires that the label is already in the message or in any of the changes being imported. The label in the message takes priority over the ones in the list of original messages of changes imported. + + +`transformation metadata.replace_message(text, ignore_label_not_found=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +text | `string`

The template text to use for the message. For example '[Import of foo ${LABEL}]'. This would construct a message resolving ${LABEL} to the corresponding label.

+ignore_label_not_found | `boolean`

If a label used in the template is not found, ignore the error and don't add the header. By default it will stop the migration and fail.

+ + +#### Example: + + +##### Replace the message: + +Replace the original message with a text: + +```python +metadata.replace_message("COPYBARA CHANGE: Import of ${GITHUB_PR_NUMBER}\n\n${GITHUB_PR_BODY}\n") +``` + +Will transform the message to: + +``` +COPYBARA CHANGE: Import of 12345 +Body from Github Pull Request +``` + + + + + +### metadata.restore_author + +For a given change, restore the author present in the ORIGINAL_AUTHOR label as the author of the change. + +`transformation metadata.restore_author(label='ORIGINAL_AUTHOR', search_all_changes=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +label | `string`

The label to use for restoring the author

+search_all_changes | `boolean`

By default Copybara only looks in the last current change for the author label. This allows to do the search in all current changes (Only makes sense for SQUASH/CHANGE_REQUEST).

+ + +### metadata.save_author + +For a given change, store a copy of the author as a label with the name ORIGINAL_AUTHOR. + +`transformation metadata.save_author(label='ORIGINAL_AUTHOR')` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +label | `string`

The label to use for storing the author

+ + +### metadata.scrubber + +Removes part of the change message using a regex + +`transformation metadata.scrubber(regex, msg_if_no_match=None, fail_if_no_match=False, replacement='')` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +regex | `string`

Any text matching the regex will be removed. Note that the regex is runs in multiline mode.

+msg_if_no_match | `string`

If set, Copybara will use this text when the scrubbing regex doesn't match.

+fail_if_no_match | `boolean`

If set, msg_if_no_match must be None and then fail if the scrubbing regex doesn't match.

+replacement | `string`

Text replacement for the matching substrings. References to regex group numbers can be used in the form of $1, $2, etc.

+ + +#### Examples: + + +##### Remove from a keyword to the end of the message: + +When change messages are in the following format: + +``` +Public change description + +This is a public description for a commit + +CONFIDENTIAL: +This fixes internal project foo-bar +``` + +Using the following transformation: + +```python +metadata.scrubber('(^|\n)CONFIDENTIAL:(.|\n)*') +``` + +Will remove the confidential part, leaving the message as: + +``` +Public change description + +This is a public description for a commit +``` + + + + +##### Keep only message enclosed in tags: + +The previous example is prone to leak confidential information since a developer could easily forget to include the CONFIDENTIAL label. A different approach for this is to scrub everything by default except what is explicitly allowed. For example, the following scrubber would remove anything not enclosed in tags: + + +```python +metadata.scrubber('^(?:\n|.)*((?:\n|.)*)(?:\n|.)*$', replacement = '$1') +``` + +So a message like: + +``` +this +is +very confidentialbut this is public +very public + +and this is a secret too +``` + +would be transformed into: + +``` +but this is public +very public +``` + + + + +##### Use default msg when the scrubbing regex doesn't match: + +Assign msg_if_no_match a default msg. For example: + + +```python +metadata.scrubber('^(?:\n|.)*((?:\n|.)*)(?:\n|.)*$', msg_if_no_match = 'Internal Change.', replacement = '$1') +``` + +So a message like: + +``` +this +is +very confidential +This is not public msg. + +and this is a secret too +``` + +would be transformed into: + +``` +Internal Change. +``` + + + + +##### Fail if the scrubbing regex doesn't match: + +Set fail_if_no_match to true + +```python +metadata.scrubber('^(?:\n|.)*((?:\n|.)*)(?:\n|.)*$', fail_if_no_match = True, replacement = '$1') +``` + +So a message like: + +``` +this +is +very confidential +but this is not public + +and this is a secret too + +``` + +This would fail. Error msg: + +``` +Scrubber regex: '^(?:\n|.)*((?:\n|.)*)(?:\n|.)*$' didn't match for description: this +is +very confidential +but this is not public + +and this is a secret too +``` + + + + + +### metadata.squash_notes + +Generate a message that includes a constant prefix text and a list of changes included in the squash change. + +`transformation metadata.squash_notes(prefix='Copybara import of the project:\n\n', max=100, compact=True, show_ref=True, show_author=True, show_description=True, oldest_first=False, use_merge=True)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +prefix | `string`

A prefix to be printed before the list of commits.

+max | `integer`

Max number of commits to include in the message. For the rest a comment like (and x more) will be included. By default 100 commits are included.

+compact | `boolean`

If compact is set, each change will be shown in just one line

+show_ref | `boolean`

If each change reference should be present in the notes

+show_author | `boolean`

If each change author should be present in the notes

+show_description | `boolean`

If each change description should be present in the notes

+oldest_first | `boolean`

If set to true, the list shows the oldest changes first. Otherwise it shows the changes in descending order.

+use_merge | `boolean`

If true then merge changes are included in the squash notes

+ + +#### Examples: + + +##### Simple usage: + +'Squash notes' default is to print one line per change with information about the author + +```python +metadata.squash_notes("Changes for Project Foo:\n") +``` + +This transform will generate changes like: + +``` +Changes for Project Foo: + + - 1234abcde second commit description by Foo Bar + - a4321bcde first commit description by Foo Bar +``` + + + +##### Removing authors and reversing the order: + + + +```python +metadata.squash_notes("Changes for Project Foo:\n", + oldest_first = True, + show_author = False, +) +``` + +This transform will generate changes like: + +``` +Changes for Project Foo: + + - a4321bcde first commit description + - 1234abcde second commit description +``` + + + +##### Removing description: + + + +```python +metadata.squash_notes("Changes for Project Foo:\n", + show_description = False, +) +``` + +This transform will generate changes like: + +``` +Changes for Project Foo: + + - a4321bcde by Foo Bar + - 1234abcde by Foo Bar +``` + + + +##### Showing the full message: + + + +```python +metadata.squash_notes( + prefix = 'Changes for Project Foo:', + compact = False +) +``` + +This transform will generate changes like: + +``` +Changes for Project Foo: +-- +2 by Foo Baz : + +second commit + +Extended text +-- +1 by Foo Bar : + +first commit + +Extended text +``` + + + + +### metadata.use_last_change + +Use metadata (message or/and author) from the last change being migrated. Useful when using 'SQUASH' mode but user only cares about the last change. + +`transformation metadata.use_last_change(author=True, message=True, default_message=None, use_merge=True)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +author | `boolean`

Replace author with the last change author (Could still be the default author if not whitelisted or using `authoring.overwrite`.

+message | `boolean`

Replace message with last change message.

+default_message | `string`

Replace message with last change message.

+use_merge | `boolean`

If true then merge changes are taken into account for looking for the last change.

+ + +### metadata.verify_match + +Verifies that a RegEx matches (or not matches) the change message. Does not transform anything, but will stop the workflow if it fails. + +`transformation metadata.verify_match(regex, verify_no_match=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +regex | `string`

The regex pattern to verify. The re2j pattern will be applied in multiline mode, i.e. '^' refers to the beginning of a file and '$' to its end.

+verify_no_match | `boolean`

If true, the transformation will verify that the RegEx does not match.

+ + +#### Example: + + +##### Check that a text is present in the change description: + +Check that the change message contains a text enclosed in : + +```python +metadata.verify_match("(.|\n)*") +``` + + + + +## origin_ref + +Reference to the change/review in the origin. + + +#### Fields: + +Name | Description +---- | ----------- +ref | Origin reference ref + + + +## patch + +Module for applying patches. + + +### patch.apply + +A transformation that applies the given patch files. If a path does not exist in a patch, it will be ignored. + +`patchTransformation patch.apply(patches=[], excluded_patch_paths=[], series=None, strip=1)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +patches | `object`

The list of patchfiles to apply, relative to the current config file.The files will be applied relative to the checkout dir and the leading pathcomponent will be stripped (-p1).

This field can be combined with 'series'. Both 'patches' and 'series' will be applied in order (patches first). **This field doesn't accept a glob**

+excluded_patch_paths | `sequence of string`

The list of paths to exclude from each of the patches. Each of the paths will be excluded from all the patches. Note that these are not workdir paths, but paths relative to the patch itself. If not empty, the patch will be applied using 'git apply' instead of GNU Patch.

+series | `string`

The config file that contains a list of patches to apply. The series file contains names of the patch files one per line. The names of the patch files are relative to the series config file. The files will be applied relative to the checkout dir and the leading path component will be stripped (-p1).:
:
This field can be combined with 'patches'. Both 'patches' and 'series' will be applied in order (patches first).

+strip | `integer`

Number of segments to strip. (This sets -pX flag, for example -p0, -p1, etc.).By default it uses -p1

+ + + +**Command line flags:** + +Name | Type | Description +---- | ---- | ----------- +`--patch-bin` | *string* | Path for GNU Patch command +`--patch-skip-version-check` | *boolean* | Skip checking the version of patch and assume it is fine +`--patch-use-git-apply` | *boolean* | Don't use GNU Patch and instead use 'git apply' + + + +## Path + +Represents a path in the checkout directory + + +#### Fields: + +Name | Description +---- | ----------- +attr | Get the file attributes, for example size. +name | Filename of the path. For foo/bar/baz.txt it would be baz.txt +parent | Get the parent path +path | Full path relative to the checkout directory + + +### path.read_symlink + +Read the symlink + +`Path path.read_symlink()` + + +### path.relativize + +Constructs a relative path between this path and a given path. For example:
path('a/b').relativize('a/b/c/d')
returns 'c/d' + +`Path path.relativize(other)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +other | `Path`

The path to relativize against this path

+ + +### path.resolve + +Resolve the given path against this path. + +`Path path.resolve(child)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +child | `object`

Resolve the given path against this path. The parameter can be a string or a Path.

+ + +### path.resolve_sibling + +Resolve the given path against this path. + +`Path path.resolve_sibling(other)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +other | `object`

Resolve the given path against this path. The parameter can be a string or a Path.

+ + + +## PathAttributes + +Represents a path attributes like size. + + +#### Fields: + +Name | Description +---- | ----------- +size | The size of the file. Throws an error if file size > 2GB. +symlink | Returns true if it is a symlink + + + +## SetReviewInput + +Input for posting a review to Gerrit. See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input + + + +## TransformWork + +Data about the set of changes that are being migrated. It includes information about changes like: the author to be used for commit, change message, etc. You receive a TransformWork object as an argument to the transformations functions used in core.workflow + + +#### Fields: + +Name | Description +---- | ----------- +author | Author to be used in the change +changes | List of changes that will be migrated +console | Get an instance of the console to report errors or warnings +message | Message to be used in the change +params | Parameters for the function if created with core.dynamic_transform + + +### ctx.add_label + +Add a label to the end of the description + +`ctx.add_label(label, value, separator="=", hidden=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +label | `string`

The label to replace

+value | `string`

The new value for the label

+separator | `string`

The separator to use for the label

+hidden | `boolean`

Don't show the label in the message but only keep it internally

+ + +### ctx.add_or_replace_label + +Replace an existing label or add it to the end of the description + +`ctx.add_or_replace_label(label, value, separator="=")` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +label | `string`

The label to replace

+value | `string`

The new value for the label

+separator | `string`

The separator to use for the label

+ + +### ctx.add_text_before_labels + +Add a text to the description before the labels paragraph + +`ctx.add_text_before_labels(text)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +text | `string`

+ + +### ctx.create_symlink + +Create a symlink + +`ctx.create_symlink(link, target)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +link | `Path`

The link path

+target | `Path`

The target path

+ + +### ctx.destination_api + +Returns an api handle for the destination repository. Methods available depend on the destination type. Use with extreme caution, as external calls can make workflow non-deterministic and possibly irreversible. Can have side effects in dry-runmode. + +`endpoint ctx.destination_api()` + + +### ctx.destination_reader + +Returns a handle to read files from the destination, if supported by the destination. + +`destination_reader ctx.destination_reader()` + + +### ctx.find_all_labels + +Tries to find all the values for a label. First it looks at the generated message (IOW labels that might have been added by previous steps), then looks in all the commit messages being imported and finally in the resolved reference passed in the CLI. + +`sequence of string ctx.find_all_labels(message)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +message | `string`

+ + +### ctx.find_label + +Tries to find a label. First it looks at the generated message (IOW labels that might have been added by previous steps), then looks in all the commit messages being imported and finally in the resolved reference passed in the CLI. + +`string ctx.find_label(label)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +label | `string`

+ + +### ctx.new_path + +Create a new path + +`Path ctx.new_path(path)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +path | `string`

The string representing the path

+ + +### ctx.now_as_string + +Get current date as a string + +`string ctx.now_as_string(format="yyyy-MM-dd", zone="UTC")` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +format | `string`

The format to use. See: https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html for details.

+zone | `object`

The timezone id to use. See https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html. By default UTC

+ + +### ctx.origin_api + +Returns an api handle for the origin repository. Methods available depend on the origin type. Use with extreme caution, as external calls can make workflow non-deterministic and possibly irreversible. Can have side effects in dry-runmode. + +`endpoint ctx.origin_api()` + + +### ctx.read_path + +Read the content of path as UTF-8 + +`string ctx.read_path(path)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +path | `Path`

The string representing the path

+ + +### ctx.remove_label + +Remove a label from the message if present + +`ctx.remove_label(label, whole_message=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +label | `string`

The label to delete

+whole_message | `boolean`

By default Copybara only looks in the last paragraph for labels. This flagmake it replace labels in the whole message.

+ + +### ctx.replace_label + +Replace a label if it exist in the message + +`ctx.replace_label(label, value, separator="=", whole_message=False)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +label | `string`

The label to replace

+value | `string`

The new value for the label

+separator | `string`

The separator to use for the label

+whole_message | `boolean`

By default Copybara only looks in the last paragraph for labels. This flagmake it replace labels in the whole message.

+ + +### ctx.run + +Run a glob or a transform. For example:
files = ctx.run(glob(['**.java']))
or
ctx.run(core.move("foo", "bar"))
or
+ +`object ctx.run(runnable)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +runnable | `object`

A glob or a transform (Transforms still not implemented)

+ + +### ctx.set_author + +Update the author to be used in the change + +`ctx.set_author(author)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +author | `author`

+ + +### ctx.set_message + +Update the message to be used in the change + +`ctx.set_message(message)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +message | `string`

+ + +### ctx.write_path + +Write an arbitrary string to a path (UTF-8 will be used) + +`ctx.write_path(path, content)` + + +#### Parameters: + +Parameter | Description +--------- | ----------- +path | `Path`

The string representing the path

+content | `string`

The content of the file

+ + diff --git a/third_party/copybara/java/com/google/copybara/BUILD b/third_party/copybara/java/com/google/copybara/BUILD new file mode 100644 index 0000000000..3940bf0837 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/BUILD @@ -0,0 +1,219 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:public"]) + +load(":docs.bzl", "doc_generator") + +exports_files( + [ + "doc_skylark.sh", + "docs.bzl", + ], + visibility = ["//visibility:public"], +) + +JAVACOPTS = [ + "-Xlint:unchecked", + "-source", + "1.8", +] + +java_binary( + name = "copybara", + javacopts = JAVACOPTS, + main_class = "com.google.copybara.Main", + runtime_deps = [ + ":copybara_main", + ], +) + +java_library( + name = "copybara_main", + srcs = ["Main.java"], + javacopts = JAVACOPTS, + deps = [ + ":base", + ":copybara_lib", + ":general_options", + "//java/com/google/copybara/config:base", + "//java/com/google/copybara/exception", + "//java/com/google/copybara/jcommander:converters", + "//java/com/google/copybara/profiler", + "//java/com/google/copybara/util", + "//java/com/google/copybara/util/console", + "//third_party:flogger", + "//third_party:guava", + "//third_party:jcommander", + "//third_party:jsr305", + "//third_party:skylark-lang", + ], +) + +doc_generator( + name = "docs", + deps = [":copybara"], +) + +BASE_SRCS = [ + "BaselinesWithoutLabelVisitor.java", + "Change.java", + "ChangeMessage.java", + "Changes.java", + "ChangeVisitable.java", + "CheckoutPath.java", + "CheckoutPathAttributes.java", + "ConfigItemDescription.java", + "Destination.java", + "DestinationEffect.java", + "DestinationReader.java", + "DestinationStatusVisitor.java", + "Endpoint.java", + "EndpointProvider.java", + "Info.java", + "LazyResourceLoader.java", + "LabelFinder.java", + "LocalParallelizer.java", + "Metadata.java", + "MigrationInfo.java", + "NonReversibleValidationException.java", + "Option.java", + "Options.java", + "Origin.java", + "Revision.java", + "SkylarkContext.java", + "Transformation.java", + "TransformResult.java", + "TransformWork.java", + "Trigger.java", + "treestate/FileSystemTreeState.java", + "treestate/MapBasedTreeState.java", + "treestate/TreeState.java", + "treestate/TreeStateUtil.java", + "WorkflowOptions.java", + "WriterContext.java", +] + +java_library( + name = "options", + srcs = [ + "Option.java", + "Options.java", + ], + javacopts = JAVACOPTS, + deps = [ + "//third_party:guava", + "//third_party:jsr305", + "//third_party:re2j", + "//third_party:shell", + ], +) + +java_library( + name = "moduleset", + srcs = ["ModuleSet.java"], + javacopts = JAVACOPTS, + deps = [ + ":options", + "//third_party:guava", + ], +) + +java_library( + name = "base", + srcs = BASE_SRCS, + javacopts = JAVACOPTS, + deps = [ + "//java/com/google/copybara/authoring", + "//java/com/google/copybara/doc:annotations", + "//java/com/google/copybara/exception", + "//java/com/google/copybara/jcommander:converters", + "//java/com/google/copybara/jcommander:validators", + "//java/com/google/copybara/util", + "//java/com/google/copybara/util/console", + "//third_party:autovalue", + "//third_party:flogger", + "//third_party:guava", + "//third_party:jcommander", + "//third_party:jsr305", + "//third_party:re2j", + "//third_party:skylark-lang", + ], +) + +java_library( + name = "general_options", + srcs = ["GeneralOptions.java"], + javacopts = JAVACOPTS, + deps = [ + ":base", + "//java/com/google/copybara/exception", + "//java/com/google/copybara/jcommander:converters", + "//java/com/google/copybara/monitor", + "//java/com/google/copybara/profiler", + "//java/com/google/copybara/util", + "//java/com/google/copybara/util/console", + "//third_party:autovalue", + "//third_party:flogger", + "//third_party:guava", + "//third_party:jcommander", + "//third_party:jsr305", + "//third_party:re2j", + "//third_party:skylark-lang", + ], +) + +java_library( + name = "copybara_lib", + srcs = glob( + ["**/*.java"], + exclude = [ + "Main.java", + "GeneralOptions.java", + ] + BASE_SRCS, + ), + javacopts = JAVACOPTS, + deps = [ + ":base", + ":general_options", + "//java/com/google/copybara/authoring", + "//java/com/google/copybara/buildozer", + "//java/com/google/copybara/buildozer:buildozer_options", + "//java/com/google/copybara/config:base", + "//java/com/google/copybara/config:global_migrations", + "//java/com/google/copybara/config:parser", + "//java/com/google/copybara/doc:annotations", + "//java/com/google/copybara/exception", + "//java/com/google/copybara/format", + "//java/com/google/copybara/git", + "//java/com/google/copybara/hg", + "//java/com/google/copybara/monitor", + "//java/com/google/copybara/profiler", + "//java/com/google/copybara/remotefile", + "//java/com/google/copybara/templatetoken", + "//java/com/google/copybara/transform", + "//java/com/google/copybara/transform/debug", + "//java/com/google/copybara/transform/patch", + "//java/com/google/copybara/util", + "//java/com/google/copybara/util/console", + "//third_party:flogger", + "//third_party:guava", + "//third_party:jcommander", + "//third_party:jsr305", + "//third_party:re2j", + "//third_party:skylark-lang", + ], +) diff --git a/third_party/copybara/java/com/google/copybara/BaselinesWithoutLabelVisitor.java b/third_party/copybara/java/com/google/copybara/BaselinesWithoutLabelVisitor.java new file mode 100644 index 0000000000..327c9143ab --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/BaselinesWithoutLabelVisitor.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.copybara.ChangeVisitable.ChangesVisitor; +import com.google.copybara.ChangeVisitable.VisitResult; +import com.google.copybara.util.Glob; +import java.util.ArrayList; +import java.util.List; + +/** A visitor that finds all the parents that match the origin glob. */ +public class BaselinesWithoutLabelVisitor implements ChangesVisitor { + + private final List result = new ArrayList<>(); + private final int limit; + private final Glob originFiles; + private boolean skipFirst; + + public BaselinesWithoutLabelVisitor(Glob originFiles, int limit, boolean skipFirst) { + this.originFiles = Preconditions.checkNotNull(originFiles); + Preconditions.checkArgument(limit > 0); + this.limit = limit; + this.skipFirst = skipFirst; + } + + public ImmutableList getResult() { + return ImmutableList.copyOf(result); + } + + @SuppressWarnings("unchecked") + @Override + public VisitResult visit(Change change) { + if (skipFirst) { + skipFirst = false; + return VisitResult.CONTINUE; + } + ImmutableSet files = change.getChangeFiles(); + if (Glob.affectsRoots(originFiles.roots(), files)) { + result.add((T) change.getRevision()); + return result.size() < limit ? VisitResult.CONTINUE : VisitResult.TERMINATE; + } + // This change only contains files that are not exported + return VisitResult.CONTINUE; + } +} diff --git a/third_party/copybara/java/com/google/copybara/Change.java b/third_party/copybara/java/com/google/copybara/Change.java new file mode 100644 index 0000000000..34879481eb --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Change.java @@ -0,0 +1,236 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import static java.time.format.DateTimeFormatter.ISO_OFFSET_DATE_TIME; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.copybara.DestinationEffect.OriginRef; +import com.google.copybara.authoring.Author; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.Dict; +import com.google.devtools.build.lib.syntax.Sequence; +import com.google.devtools.build.lib.syntax.StarlarkList; +import com.google.devtools.build.lib.syntax.StarlarkValue; +import java.time.ZonedDateTime; +import java.util.Objects; +import java.util.Set; +import javax.annotation.Nullable; + +/** Represents a change in a Repository */ +@StarlarkBuiltin( + name = "change", + category = StarlarkDocumentationCategory.BUILTIN, + doc = "A change metadata. Contains information like author, change message or detected labels") +public final class Change extends OriginRef implements StarlarkValue { + + private final R revision; + private final Author author; + private final String message; + private final ZonedDateTime dateTime; + private final ImmutableListMultimap labels; + private Author mappedAuthor; + private final boolean merge; + @Nullable + private final ImmutableList parents; + + @Nullable + private final ImmutableSet changeFiles; + + public Change(R revision, Author author, String message, ZonedDateTime dateTime, + ImmutableListMultimap labels) { + this(revision, author, message, dateTime, labels, /*changeFiles=*/null); + } + + public Change(R revision, Author author, String message, ZonedDateTime dateTime, + ImmutableListMultimap labels, @Nullable Set changeFiles) { + this(revision, author, message, dateTime, labels, changeFiles, /*merge=*/false, + /*parents=*/ null); + } + + public Change(R revision, Author author, String message, ZonedDateTime dateTime, + ImmutableListMultimap labels, @Nullable Set changeFiles, + boolean merge, @Nullable ImmutableList parents) { + super(revision.asString()); + this.revision = Preconditions.checkNotNull(revision); + this.author = Preconditions.checkNotNull(author); + this.message = Preconditions.checkNotNull(message); + this.dateTime = dateTime; + this.labels = labels; + this.changeFiles = changeFiles == null ? null : ImmutableSet.copyOf(changeFiles); + this.merge = merge; + this.parents = parents; + } + + /** + * Reference of the change. For example a SHA-1 reference in git. + */ + public R getRevision() { + return revision; + } + + /** + * Return the parent revisions if the origin provides that information. Currently only for Git and + * Hg. Otherwise null. + */ + @Nullable + public ImmutableList getParents() { + return parents; + } + + @StarlarkMethod(name = "original_author", doc = "The author of the change before any" + + " mapping", structField = true) + public Author getAuthor() { + return author; + } + + /** + * The author of the change. Can already be mapped using metadata.map_author + */ + @StarlarkMethod(name = "author", doc = "The author of the change", structField = true) + public Author getMappedAuthor() { + return Preconditions.checkNotNull(mappedAuthor == null ? author : mappedAuthor); + } + + public void setMappedAuthor(Author mappedAuthor) { + this.mappedAuthor = mappedAuthor; + } + + @StarlarkMethod(name = "message", doc = "The message of the change", structField = true) + public String getMessage() { + return message; + } + + @StarlarkMethod( + name = "labels", + doc = + "A dictionary with the labels detected for the change. If the label is present multiple" + + " times it returns the last value. Note that this is a heuristic and it could" + + " include things that are not labels.", + structField = true) + public Dict getLabelsForSkylark() { + return Dict.copyOf( + /* mu= */ null, + ImmutableMap.copyOf(Maps.transformValues(labels.asMap(), Iterables::getLast))); + } + + @StarlarkMethod( + name = "labels_all_values", + doc = + "A dictionary with the labels detected for the change. Note that the value is a" + + " collection of the values for each time the label was found. Use 'labels' instead" + + " if you are only interested in the last value. Note that this is a heuristic and" + + " it could include things that are not labels.", + structField = true) + public Dict> getLabelsAllForSkylark() { + return Dict.copyOf( + /* mu= */ null, Maps.transformValues(labels.asMap(), StarlarkList::immutableCopyOf)); + } + + /** + * If not null, the files that were affected in this change. + */ + @Nullable + public ImmutableSet getChangeFiles() { + return changeFiles; + } + + public ZonedDateTime getDateTime() { + return dateTime; + } + + @StarlarkMethod(name = "date_time_iso_offset", + doc = "Return a ISO offset date time. Example: 2011-12-03T10:15:30+01:00'", + structField = true) + public String dateTimeFmt() { + return ISO_OFFSET_DATE_TIME.format(getDateTime()); + } + + public ImmutableListMultimap getLabels() { + return labels; + } + + /** + * Returns the first line of the change. Usually a summary. + */ + @StarlarkMethod(name = "first_line_message", doc = "The message of the change" + , structField = true) + public String firstLineMessage() { + return extractFirstLine(message); + } + + static String extractFirstLine(String message) { + int idx = message.indexOf('\n'); + return idx == -1 ? message : message.substring(0, idx); + } + + /** + * Returns true if the change represents a merge. + */ + @StarlarkMethod(name = "merge", doc = "Returns true if the change represents a merge" + , structField = true) + public boolean isMerge() { + return merge; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("revision", revision.asString()) + .add("author", author) + .add("dateTime", dateTime) + .add("message", message) + .add("merge", merge) + .add("parents", parents) + .toString(); + } + + public Change withLabels(ImmutableListMultimap newLabels) { + return new Change<>(revision, author, message, dateTime, + Revision.addNewLabels(labels, newLabels), changeFiles, merge, parents); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Change change = (Change) o; + return Objects.equals(revision, change.revision) + && Objects.equals(author, change.author) + && Objects.equals(message, change.message) + && Objects.equals(dateTime, change.dateTime) + && Objects.equals(labels, change.labels); + } + + @Override + public int hashCode() { + return Objects.hash(revision, author, message, dateTime, labels); + } +} diff --git a/third_party/copybara/java/com/google/copybara/ChangeMessage.java b/third_party/copybara/java/com/google/copybara/ChangeMessage.java new file mode 100644 index 0000000000..faae8a7d25 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/ChangeMessage.java @@ -0,0 +1,253 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.copybara.doc.annotations.DocSignaturePrefix; +import com.google.devtools.build.lib.skylarkinterface.Param; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.Sequence; +import com.google.devtools.build.lib.syntax.StarlarkList; +import com.google.devtools.build.lib.syntax.StarlarkValue; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.CheckReturnValue; + +/** + * An object that represents a well formed message: No superfluous new lines, a group of labels, + * etc. + * + *

This class is immutable. + */ +@SuppressWarnings("unused") +@StarlarkBuiltin( + name = "ChangeMessage", + category = StarlarkDocumentationCategory.BUILTIN, + doc = "Represents a well formed parsed change message with its associated labels.") +@DocSignaturePrefix("message") +public final class ChangeMessage implements StarlarkValue { + + private static final String DOUBLE_NEWLINE = "\n\n"; + private static final String DASH_DASH_SEPARATOR = "\n--\n"; + private static final CharMatcher TRIM = CharMatcher.is('\n'); + + private final String text; + private final String groupSeparator; + private final ImmutableList labels; + + private ChangeMessage(String text, String groupSeparator, List labels) { + this.text = TRIM.trimFrom(text); + this.groupSeparator = Preconditions.checkNotNull(groupSeparator); + this.labels = ImmutableList.copyOf(Preconditions.checkNotNull(labels)); + } + + /** + * Create a new message object looking for labels in just the last paragraph. + * + *

Use this for Copybara well-formed messages. + */ + public static ChangeMessage parseMessage(String message) { + String trimMsg = TRIM.trimFrom(message); + int doubleNewLine = trimMsg.lastIndexOf(DOUBLE_NEWLINE); + int dashDash = trimMsg.lastIndexOf(DASH_DASH_SEPARATOR); + if (doubleNewLine == -1 && dashDash == -1) { + // Empty message like "\n\nfoo: bar" or "\n\nfoo bar baz" + if (message.startsWith(DOUBLE_NEWLINE)) { + return new ChangeMessage("", DOUBLE_NEWLINE, linesAsLabels(trimMsg)); + } + return new ChangeMessage(trimMsg, DOUBLE_NEWLINE, new ArrayList<>()); + } else if (doubleNewLine > dashDash) { + return new ChangeMessage(trimMsg.substring(0, doubleNewLine), DOUBLE_NEWLINE, + linesAsLabels(trimMsg.substring(doubleNewLine + 2))); + } else { + return new ChangeMessage(trimMsg.substring(0, dashDash), DASH_DASH_SEPARATOR, + linesAsLabels(trimMsg.substring(dashDash + 4))); + } + } + + /** + * Create a new message object treating all the lines as possible labels instead of looking + * just in the last paragraph for labels. + */ + public static ChangeMessage parseAllAsLabels(String message) { + Preconditions.checkNotNull(message); + return new ChangeMessage("", DOUBLE_NEWLINE, linesAsLabels(message)); + } + + private static List linesAsLabels(String message) { + Preconditions.checkNotNull(message); + return Splitter.on('\n').splitToList(TRIM.trimTrailingFrom(message)).stream() + .map(LabelFinder::new) + .collect(Collectors.toList()); + } + + @StarlarkMethod(name = "first_line", doc = "First line of this message", structField = true) + public String firstLine() { + int idx = text.indexOf('\n'); + return idx == -1 ? text : text.substring(0, idx); + } + + @StarlarkMethod( + name = "text", + doc = "The text description this message, not including the labels.", + structField = true) + public String getText() { + return text; + } + + public ImmutableList getLabels() { + return labels; + } + + /** + * Returns all the labels in the message. If a label appears multiple times, it respects the + * order of appearance. + */ + public ImmutableListMultimap labelsAsMultimap(){ + // We overwrite duplicates + ImmutableListMultimap.Builder result = ImmutableListMultimap.builder(); + for (LabelFinder label : labels) { + if (label.isLabel()) { + result.put(label.getName(), label.getValue()); + } + } + return result.build(); + } + + @StarlarkMethod( + name = "label_values", + doc = "Returns a list of values associated with the label name.", + parameters = { + @Param(name = "label_name", type = String.class, named = true, doc = "The label name."), + }) + public Sequence getLabelValues(String labelName) { + ImmutableListMultimap localLabels = labelsAsMultimap(); + if (localLabels.containsKey(labelName)) { + return StarlarkList.immutableCopyOf(localLabels.get(labelName)); + } + return StarlarkList.empty(); + } + + + @CheckReturnValue + public ChangeMessage withLabel(String name, String separator, String value) { + List newLabels = new ArrayList<>(labels); + // Add an additional line if none of the previous elements are labels + if (!newLabels.isEmpty() && newLabels.stream().noneMatch(LabelFinder::isLabel)) { + newLabels.add(new LabelFinder("")); + } + newLabels.add(new LabelFinder(validateLabelName(name) + Preconditions + .checkNotNull(separator) + Preconditions.checkNotNull(value))); + return new ChangeMessage(this.text, this.groupSeparator, newLabels); + } + + @CheckReturnValue + public ChangeMessage withReplacedLabel(String labelName, String separator , String value) { + validateLabelName(labelName); + List newLabels = labels.stream().map(label -> label.isLabel(labelName) + ? new LabelFinder(labelName + separator + value) + : label) + .collect(Collectors.toList()); + return new ChangeMessage(this.text, this.groupSeparator, newLabels); + } + + @CheckReturnValue + public ChangeMessage withNewOrReplacedLabel(String labelName, String separator, String value) { + validateLabelName(labelName); + List newLabels = new ArrayList<>(); + boolean wasReplaced = false; + + for (LabelFinder originalLabel : labels) { + if (originalLabel.isLabel(labelName)) { + newLabels.add(new LabelFinder(labelName + separator + value)); + wasReplaced = true; + } else { + newLabels.add(originalLabel); + } + } + + ChangeMessage newChangeMessage = new ChangeMessage(this.text, this.groupSeparator, newLabels); + if (!wasReplaced) { + return newChangeMessage.withLabel(labelName, separator, value); + } + return newChangeMessage; + } + + /** + * Remove a label by name if it exist. + */ + @CheckReturnValue + public ChangeMessage withRemovedLabelByName(String name) { + validateLabelName(name); + ImmutableList filteredLabels = + labels + .stream() + .filter(label -> !label.isLabel(name)) + .collect(ImmutableList.toImmutableList()); + return new ChangeMessage(this.text, this.groupSeparator, filteredLabels); + } + + /** + * Remove a label by name and value if it exist. + */ + @CheckReturnValue + public ChangeMessage withRemovedLabelByNameAndValue(String name, String value) { + validateLabelName(name); + ImmutableList filteredLabels = + labels + .stream() + .filter(label -> !label.isLabel(name) || !label.getValue().equals(value)) + .collect(ImmutableList.toImmutableList()); + return new ChangeMessage(this.text, this.groupSeparator, filteredLabels); + } + + private static String validateLabelName(String label) { + Preconditions.checkArgument(LabelFinder.VALID_LABEL.matcher(label).matches(), + "Label '%s' is not a valid label", label); + return label; + } + + /** + * Set the text part of the message, leaving the labels untouched.L + */ + @CheckReturnValue + public ChangeMessage withText(String text) { + return new ChangeMessage(TRIM.trimFrom(text), this.groupSeparator, this.labels); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + if (!text.isEmpty()) { + sb.append(text).append(labels.isEmpty() ? "\n" : groupSeparator); + } + for (LabelFinder label : labels) { + sb.append(label.getLine()).append('\n'); + } + // Lets normalize in case parseAllAsLabels was used and all the labels where + // removed. + return TRIM.trimFrom(sb.toString()) + '\n'; + } +} diff --git a/third_party/copybara/java/com/google/copybara/ChangeVisitable.java b/third_party/copybara/java/com/google/copybara/ChangeVisitable.java new file mode 100644 index 0000000000..05da35cc24 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/ChangeVisitable.java @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * An interface stating that the implementing class accepts child visitors to explore repository + * state beyond the changes being migrated. + */ +public interface ChangeVisitable { + + /** + * Visit the parents of the {@code start} revision and call the visitor for each + * change. The visitor can stop the stream of changes at any moment by returning {@see + * VisitResult#TERMINATE}. + * + *

It is up to the Origin how and what changes it provides to the function. + */ + void visitChanges(@Nullable R start, ChangesVisitor visitor) + throws RepoException, ValidationException; + + /** + * Visit only changes that contain any of the labels in {@code labels}. + */ + default void visitChangesWithAnyLabel( + @Nullable R start, ImmutableCollection labels, ChangesLabelVisitor visitor) + throws RepoException, ValidationException { + visitChanges(start, input -> { + // We could return all the label values, but this is really only used for + // RevId like ones and last is good enough for now. + Map copy = Maps.newHashMap(Maps.transformValues(input.getLabels().asMap(), + Iterables::getLast)); + copy.keySet().retainAll(labels); + if (copy.isEmpty()) { + return VisitResult.CONTINUE; + } + return visitor.visit(input, ImmutableMap.copyOf(copy)); + }); + } + + /** + * A visitor of changes. An implementation of this interface is provided to {@see + * visitChanges} methods to visit changes in Origin or + * Destination history. + */ + interface ChangesVisitor { + + /** + * Invoked for each change found. The implementation can chose to cancel the visitation by + * returning {@link VisitResult#TERMINATE}. + */ + VisitResult visit(Change input); + } + + /** + * A visitor of changes that only receives changes that match any of the passed labels. + */ + interface ChangesLabelVisitor { + + /** + * Invoked for each change found that matches the labels. + * + *

Note that the {@code matchedLabels} can be disjoint with the labels in {@code input}, + * since labels might be stored with a different string format. + */ + VisitResult visit(Change input, ImmutableMap matchedLabels); + } + + /** + * The result type for the function passed to + * {@see visitChanges}. + */ + enum VisitResult { + /** + * Continue. If more changes are available for visiting, the origin will call again the + * function with the next changes. + */ + CONTINUE, + /** + * Stop. Origin will not pass more changes to the visitor function. Usually used because the + * function found what it was looking for (For example a commit with a label). + */ + TERMINATE + } +} + diff --git a/third_party/copybara/java/com/google/copybara/Changes.java b/third_party/copybara/java/com/google/copybara/Changes.java new file mode 100644 index 0000000000..0e79ee1fc2 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Changes.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.Sequence; +import com.google.devtools.build.lib.syntax.StarlarkList; +import com.google.devtools.build.lib.syntax.StarlarkValue; + +/** Information about the changes being imported */ +@StarlarkBuiltin( + name = "Changes", + category = StarlarkDocumentationCategory.BUILTIN, + doc = + "Data about the set of changes that are being migrated. " + + "Each change includes information like: original author, change message, " + + "labels, etc. You receive this as a field in TransformWork object for used defined " + + "transformations") +public final class Changes implements StarlarkValue { + + public static final Changes EMPTY = new Changes(ImmutableList.of(), ImmutableList.of()); + + private final Sequence> current; + private final Sequence> migrated; + + public Changes(Iterable> current, Iterable> migrated) { + this.current = StarlarkList.immutableCopyOf(current); + this.migrated = StarlarkList.immutableCopyOf(migrated); + } + + @StarlarkMethod( + name = "current", + doc = "List of changes that will be migrated", + structField = true) + public final Sequence> getCurrent() { + return current; + } + + @StarlarkMethod( + name = "migrated", + doc = + "List of changes that where migrated in previous Copybara executions or if using" + + " ITERATIVE mode in previous iterations of this workflow.", + structField = true) + public Sequence> getMigrated() { + return migrated; + } +} diff --git a/third_party/copybara/java/com/google/copybara/CheckoutPath.java b/third_party/copybara/java/com/google/copybara/CheckoutPath.java new file mode 100644 index 0000000000..ef63f754d4 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/CheckoutPath.java @@ -0,0 +1,218 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.common.flogger.FluentLogger; +import com.google.copybara.doc.annotations.DocSignaturePrefix; +import com.google.copybara.util.FileUtil; +import com.google.copybara.util.FileUtil.ResolvedSymlink; +import com.google.copybara.util.Glob; +import com.google.devtools.build.lib.skylarkinterface.Param; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Printer; +import com.google.devtools.build.lib.syntax.Starlark; +import com.google.devtools.build.lib.syntax.StarlarkValue; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; + +/** + * Represents a file that is exposed to Skylark. + * + *

Files are always relative to the checkout dir and normalized. + */ +@SuppressWarnings("unused") +@StarlarkBuiltin( + name = "Path", + category = StarlarkDocumentationCategory.BUILTIN, + doc = "Represents a path in the checkout directory") +@DocSignaturePrefix("path") +public class CheckoutPath implements Comparable, StarlarkValue { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private final Path path; + private final Path checkoutDir; + + CheckoutPath(Path path, Path checkoutDir) { + this.path = Preconditions.checkNotNull(path); + this.checkoutDir = Preconditions.checkNotNull(checkoutDir); + } + + private CheckoutPath create(Path path) throws EvalException { + return createWithCheckoutDir(path, checkoutDir); + } + + static CheckoutPath createWithCheckoutDir(Path relative, Path checkoutDir) throws EvalException { + if (relative.isAbsolute()) { + throw Starlark.errorf("Absolute paths are not allowed: %s", relative); + } + return new CheckoutPath(relative.normalize(), checkoutDir); + } + + @StarlarkMethod(name = "path", doc = "Full path relative to the checkout directory", + structField = true) + public String fullPath() { + return path.toString(); + } + + @StarlarkMethod(name = "name", + doc = "Filename of the path. For foo/bar/baz.txt it would be baz.txt", + structField = true) + public String name() { + return path.getFileName().toString(); + } + + @StarlarkMethod( + name = "parent", + doc = "Get the parent path", + structField = true, + allowReturnNones = true) + public Object parent() throws EvalException { + Path parent = path.getParent(); + if (parent == null) { + // nio equivalent of new_path("foo").parent returns null, but we want to be able to do + // foo.parent.resolve("bar"). While sibbling could be use for this, sometimes we'll need + // to return the parent folder and another function resolve a path based on that. + return path.toString().equals("") ? Starlark.NONE : create(path.getFileSystem().getPath("")); + } + return create(parent); + } + + @StarlarkMethod( + name = "relativize", + doc = + "Constructs a relative path between this path and a given path. For example:
" + + " path('a/b').relativize('a/b/c/d')
" + + "returns 'c/d'", + parameters = { + @Param( + name = "other", + type = CheckoutPath.class, + doc = "The path to relativize against this path"), + }) + public CheckoutPath relativize(CheckoutPath other) throws EvalException { + return create(path.relativize(other.path)); + } + + @StarlarkMethod( + name = "resolve", + doc = "Resolve the given path against this path.", + parameters = { + @Param( + name = "child", + type = Object.class, + doc = + "Resolve the given path against this path. The parameter" + + " can be a string or a Path.") + }) + public CheckoutPath resolve(Object child) throws EvalException { + if (child instanceof String) { + return create(path.resolve((String) child)); + } else if (child instanceof CheckoutPath) { + return create(path.resolve(((CheckoutPath) child).path)); + } + throw Starlark.errorf( + "Cannot resolve children for type %s: %s", child.getClass().getSimpleName(), child); + } + + @StarlarkMethod( + name = "resolve_sibling", + doc = "Resolve the given path against this path.", + parameters = { + @Param( + name = "other", + type = Object.class, + doc = + "Resolve the given path against this path. The parameter can be a string or" + + " a Path."), + }) + public CheckoutPath resolveSibling(Object other) throws EvalException { + if (other instanceof String) { + return create(path.resolveSibling((String) other)); + } else if (other instanceof CheckoutPath) { + return create(path.resolveSibling(((CheckoutPath) other).path)); + } + throw Starlark.errorf( + "Cannot resolve sibling for type %s: %s", other.getClass().getSimpleName(), other); + } + + @StarlarkMethod( + name = "attr", + doc = "Get the file attributes, for example size.", + structField = true) + public CheckoutPathAttributes attr() throws EvalException { + try { + return new CheckoutPathAttributes(path, + Files.readAttributes(checkoutDir.resolve(path), BasicFileAttributes.class, + LinkOption.NOFOLLOW_LINKS)); + } catch (IOException e) { + String msg = "Error getting attributes for " + path + ":" + e; + logger.atSevere().withCause(e).log(msg); + throw Starlark.errorf("%s", msg); // or IOException? + } + } + + @StarlarkMethod(name = "read_symlink", doc = "Read the symlink") + public CheckoutPath readSymbolicLink() throws EvalException { + try { + Path symlinkPath = checkoutDir.resolve(path); + if (!Files.isSymbolicLink(symlinkPath)) { + throw Starlark.errorf("%s is not a symlink", path); + } + + ResolvedSymlink resolvedSymlink = + FileUtil.resolveSymlink(Glob.ALL_FILES.relativeTo(checkoutDir), symlinkPath); + if (!resolvedSymlink.isAllUnderRoot()) { + throw Starlark.errorf( + "Symlink %s points to a file outside the checkout dir: %s", + symlinkPath, resolvedSymlink.getRegularFile()); + } + + return create(checkoutDir.relativize(resolvedSymlink.getRegularFile())); + } catch (IOException e) { + String msg = String.format("Cannot resolve symlink %s: %s", path, e); + logger.atSevere().withCause(e).log(msg); + throw Starlark.errorf("%s", msg); + } + } + + public Path getPath() { + return path; + } + + @Override + public String toString() { + return path.toString(); + } + + @Override + public int compareTo(CheckoutPath o) { + return this.path.compareTo(o.path); + } + + @Override + public void repr(Printer printer) { + printer.append(path.toString()); + } +} diff --git a/third_party/copybara/java/com/google/copybara/CheckoutPathAttributes.java b/third_party/copybara/java/com/google/copybara/CheckoutPathAttributes.java new file mode 100644 index 0000000000..19486eeef1 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/CheckoutPathAttributes.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.Starlark; +import com.google.devtools.build.lib.syntax.StarlarkValue; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; + +/** Represents file attributes exposed to Skylark. */ +@SuppressWarnings("unused") +@StarlarkBuiltin( + name = "PathAttributes", + category = StarlarkDocumentationCategory.BUILTIN, + doc = "Represents a path attributes like size.") +public class CheckoutPathAttributes implements StarlarkValue { + + private final Path path; + private final BasicFileAttributes attributes; + + CheckoutPathAttributes(Path path, BasicFileAttributes attributes) { + this.path = Preconditions.checkNotNull(path); + this.attributes = Preconditions.checkNotNull(attributes); + } + + @StarlarkMethod( + name = "size", + doc = "The size of the file. Throws an error if file size > 2GB.", + structField = true) + public int size() throws Exception { + long size = attributes.size(); + try { + return Math.toIntExact(size); + } catch (ArithmeticException e) { + throw Starlark.errorf("File %s is too big to compute the size: %d bytes", path, size); + } + } + + @StarlarkMethod(name = "symlink", + doc = "Returns true if it is a symlink", structField = true) + public boolean isSymlink() { + return attributes.isSymbolicLink(); + } +} diff --git a/third_party/copybara/java/com/google/copybara/CommandEnv.java b/third_party/copybara/java/com/google/copybara/CommandEnv.java new file mode 100644 index 0000000000..ee3896bd8d --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/CommandEnv.java @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.copybara.exception.CommandLineException; +import java.nio.file.Path; +import javax.annotation.Nullable; + +/** + * Environment information for command execution: arguments, workdir, etc. + */ +public class CommandEnv { + + private final Path workdir; + private final Options options; + private final ImmutableList args; + @Nullable + private ConfigFileArgs configFileArgs; + + @VisibleForTesting + public CommandEnv(Path workdir, Options options, ImmutableList args) { + this.workdir = Preconditions.checkNotNull(workdir); + this.options = Preconditions.checkNotNull(options); + this.args = Preconditions.checkNotNull(args); + } + + /** + * Get the arguments parsed as config [migration [source_ref]...] if the command uses that format. + */ + @Nullable + public ConfigFileArgs getConfigFileArgs() { + return configFileArgs; + } + + /** + * Parse the CLI arguments as config [workflow [source_ref]...] + */ + public ConfigFileArgs parseConfigFileArgs(CopybaraCmd cmd, boolean usesSourceRef) + throws CommandLineException { + Preconditions.checkState(this.configFileArgs == null, "parseConfigFileArgs was already" + + " called. Only one invocation allowed."); + if (args.isEmpty()) { + throw new CommandLineException( + String.format("Configuration file missing for '%s' subcommand.", cmd.name())); + } + + String configPath = args.get(0); + + if (args.size() < 2) { + configFileArgs = new ConfigFileArgs(configPath, /*workflowName=*/null); + return configFileArgs; + } + String workflowName = args.get(1); + if (args.size() < 3) { + configFileArgs = new ConfigFileArgs(configPath, workflowName); + return configFileArgs; + } + + if (!usesSourceRef) { + throw new CommandLineException( + String.format("Too many arguments for subcommand '%s'", cmd.name())); + } + configFileArgs = new ConfigFileArgs(configPath, workflowName, args.subList(2, args.size())); + return configFileArgs; + } + + + public Path getWorkdir() { + return workdir; + } + + public Options getOptions() { + return options; + } + + public ImmutableList getArgs() { + return args; + } +} diff --git a/third_party/copybara/java/com/google/copybara/ConfigFileArgs.java b/third_party/copybara/java/com/google/copybara/ConfigFileArgs.java new file mode 100644 index 0000000000..30250c3701 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/ConfigFileArgs.java @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import java.util.List; +import javax.annotation.Nullable; + +/** + * Arguments for a command that expects the CLI arguments be like: config_file [workflow + * [source_ref]] + */ +public final class ConfigFileArgs { + + private final String configPath; + @Nullable + private final String workflowName; + private final ImmutableList sourceRefs; + + ConfigFileArgs(String configPath, @Nullable String workflowName) { + this(configPath, workflowName, ImmutableList.of()); + } + + ConfigFileArgs(String configPath, @Nullable String workflowName, List sourceRefs) { + this.configPath = Preconditions.checkNotNull(configPath); + this.workflowName = workflowName; + this.sourceRefs = ImmutableList.copyOf(sourceRefs); + } + + public String getConfigPath() { + return configPath; + } + + public String getWorkflowName() { + return workflowName == null ? "default" : workflowName; + } + + public boolean hasWorkflowName() { + return workflowName != null; + } + + /** + * Returns the first sourceRef from the command arguments, or null if no source ref was provided. + * + *

This method is provided for convenience, for subocmmands that only care about the first + * source_ref. + */ + @Nullable + public String getSourceRef() { + return Iterables.getFirst(sourceRefs, /*default*/ null); + } + + public ImmutableList getSourceRefs() { + return sourceRefs; + } +} diff --git a/third_party/copybara/java/com/google/copybara/ConfigItemDescription.java b/third_party/copybara/java/com/google/copybara/ConfigItemDescription.java new file mode 100644 index 0000000000..64af816bba --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/ConfigItemDescription.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.collect.ImmutableSetMultimap; +import com.google.copybara.util.Glob; + +/** + * Interface for self-description. The information returned should be sufficient to create a new + * instance with identical migration behavior (but potentially different side effects). This is + * intended for discovering changes in a config. + */ +public interface ConfigItemDescription { + + default String getType() { + return getClass().getName(); + } + + /** Returns a key-value ist of the options the endpoint was instantiated with. */ + default ImmutableSetMultimap describe(Glob originFiles) { + ImmutableSetMultimap.Builder builder = + new ImmutableSetMultimap.Builder() + .put("type", getType()); + return builder.build(); + } +} diff --git a/third_party/copybara/java/com/google/copybara/ConfigLoader.java b/third_party/copybara/java/com/google/copybara/ConfigLoader.java new file mode 100644 index 0000000000..0f9d84835d --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/ConfigLoader.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.copybara.config.Config; +import com.google.copybara.config.ConfigFile; +import com.google.copybara.config.SkylarkParser; +import com.google.copybara.config.SkylarkParser.ConfigWithDependencies; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.profiler.Profiler.ProfilerTask; +import com.google.copybara.util.console.Console; +import com.google.copybara.util.console.StarlarkMode; +import java.io.IOException; + +/** + * Loads the configuration from a given config file. + */ +public class ConfigLoader { + + private final SkylarkParser skylarkParser; + private final ConfigFile configFile; + private final ModuleSet moduleSet; + + public ConfigLoader(ModuleSet moduleSet, ConfigFile configFile, StarlarkMode validateStarlark) { + this.moduleSet = moduleSet; + this.skylarkParser = new SkylarkParser(this.moduleSet.getStaticModules(), validateStarlark); + this.configFile = Preconditions.checkNotNull(configFile); + } + + /** + * Returns a string representation of the location of this configuration. + */ + public String location() { + return configFile.path(); + } + + /** + * Loads the configuration using this loader. + * @param console the console to use for reporting progress/errors + */ + public Config load(Console console) throws ValidationException, IOException { + return loadForConfigFile(console, configFile); + } + + /** + * Loads the configuration using this loader. + * @param console the console to use for reporting progress/errors + */ + public ConfigWithDependencies loadWithDependencies(Console console) + throws ValidationException, IOException { + console.progressFmt("Loading config and dependencies %s", configFile.getIdentifier()); + + try (ProfilerTask ignore = moduleSet.getOptions().get(GeneralOptions.class).profiler() + .start("loading_config_with_deps")) { + return skylarkParser.getConfigWithTransitiveImports(configFile, moduleSet, console); + } + } + + protected Config loadForConfigFile(Console console, ConfigFile configFile) + throws IOException, ValidationException { + console.progressFmt("Loading config %s", configFile.getIdentifier()); + + try (ProfilerTask ignore = moduleSet.getOptions().get(GeneralOptions.class).profiler() + .start("loading_config")) { + return skylarkParser.loadConfig(configFile, moduleSet, console); + } + } + + protected Config doLoadForRevision(Console console, Revision revision) + throws ValidationException, RepoException { + throw new UnsupportedOperationException( + "This origin/configuration doesn't allow loading configs from specific revisions"); + } + + public final Config loadForRevision(Console console, Revision revision) + throws ValidationException, RepoException { + try (ProfilerTask ignore = moduleSet.getOptions().get(GeneralOptions.class).profiler() + .start("loading_config_for_revision")) { + return doLoadForRevision(console, revision); + } + } + + public boolean supportsLoadForRevision() { + return false; + } +} diff --git a/third_party/copybara/java/com/google/copybara/ConfigLoaderProvider.java b/third_party/copybara/java/com/google/copybara/ConfigLoaderProvider.java new file mode 100644 index 0000000000..a34ba2eb14 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/ConfigLoaderProvider.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.copybara.exception.ValidationException; +import java.io.IOException; +import javax.annotation.Nullable; + +/** A class that given a main config path (copy.bara.sky file) returns a ConfigLoader. */ +public interface ConfigLoaderProvider { + + /** Create a new loader for {@code configPath} */ + ConfigLoader newLoader(String configPath, @Nullable String sourceRef) + throws ValidationException, IOException; + +} diff --git a/third_party/copybara/java/com/google/copybara/ContextProvider.java b/third_party/copybara/java/com/google/copybara/ContextProvider.java new file mode 100644 index 0000000000..1651fbf1da --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/ContextProvider.java @@ -0,0 +1,31 @@ +package com.google.copybara; + +/* + * Copyright (C) 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.google.common.collect.ImmutableMap; +import com.google.copybara.config.Config; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.util.console.Console; +import java.io.IOException; + +/** A class providing additional context for CMD*/ +public interface ContextProvider { + /** get context for CMD */ + ImmutableMap getContext(Config config, ConfigFileArgs configFileArgs, + ConfigLoaderProvider configLoaderProvider, Console console) + throws ValidationException, IOException; +} diff --git a/third_party/copybara/java/com/google/copybara/CopybaraCmd.java b/third_party/copybara/java/com/google/copybara/CopybaraCmd.java new file mode 100644 index 0000000000..f505e141a0 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/CopybaraCmd.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.util.ExitCode; +import java.io.IOException; + +/** + * A Copybara command like 'info' 'migrate', etc. + */ +public interface CopybaraCmd { + + /** + * Run the command + * @param commandEnv Command environment: Params, workdir, etc. + * @return Result exit code + */ + ExitCode run(CommandEnv commandEnv) throws ValidationException, IOException, RepoException; + + /** + * Command name + */ + String name(); + +} diff --git a/third_party/copybara/java/com/google/copybara/Core.java b/third_party/copybara/java/com/google/copybara/Core.java new file mode 100644 index 0000000000..521c20f941 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Core.java @@ -0,0 +1,1632 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import static com.google.copybara.Workflow.COPYBARA_CONFIG_PATH_IDENTITY_VAR; +import static com.google.copybara.config.GlobalMigrations.getGlobalMigrations; +import static com.google.copybara.config.SkylarkUtil.check; +import static com.google.copybara.config.SkylarkUtil.convertFromNoneable; +import static com.google.copybara.config.SkylarkUtil.stringToEnum; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.copybara.authoring.Author; +import com.google.copybara.authoring.Authoring; +import com.google.copybara.config.ConfigFile; +import com.google.copybara.config.LabelsAwareModule; +import com.google.copybara.config.Migration; +import com.google.copybara.config.SkylarkUtil; +import com.google.copybara.doc.annotations.DocDefault; +import com.google.copybara.doc.annotations.Example; +import com.google.copybara.doc.annotations.UsesFlags; +import com.google.copybara.exception.EmptyChangeException; +import com.google.copybara.feedback.Action; +import com.google.copybara.feedback.Feedback; +import com.google.copybara.feedback.SkylarkAction; +import com.google.copybara.templatetoken.Parser; +import com.google.copybara.templatetoken.Token; +import com.google.copybara.templatetoken.Token.TokenType; +import com.google.copybara.transform.CopyOrMove; +import com.google.copybara.transform.ExplicitReversal; +import com.google.copybara.transform.FilterReplace; +import com.google.copybara.transform.Remove; +import com.google.copybara.transform.Replace; +import com.google.copybara.transform.ReplaceMapper; +import com.google.copybara.transform.ReversibleFunction; +import com.google.copybara.transform.Sequence; +import com.google.copybara.transform.SkylarkTransformation; +import com.google.copybara.transform.TodoReplace; +import com.google.copybara.transform.TodoReplace.Mode; +import com.google.copybara.transform.VerifyMatch; +import com.google.copybara.transform.debug.DebugOptions; +import com.google.copybara.util.Glob; +import com.google.devtools.build.lib.skylarkinterface.Param; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.Dict; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Location; +import com.google.devtools.build.lib.syntax.Module; +import com.google.devtools.build.lib.syntax.NoneType; +import com.google.devtools.build.lib.syntax.Starlark; +import com.google.devtools.build.lib.syntax.StarlarkCallable; +import com.google.devtools.build.lib.syntax.StarlarkThread; +import com.google.devtools.build.lib.syntax.StarlarkValue; +import com.google.re2j.Pattern; +import java.util.IllegalFormatException; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * Main configuration class for creating migrations. + * + *

This class is exposed in Skylark configuration as an instance variable called "core". So users + * can use it as: + * + * 

+ * core.workspace(
+ *   name = "foo",
+ *   ...
+ * )
+ *
+ */ +@StarlarkBuiltin( + name = "core", + doc = "Core functionality for creating migrations, and basic transformations.", + category = StarlarkDocumentationCategory.BUILTIN) +@UsesFlags({GeneralOptions.class, DebugOptions.class}) +public class Core implements LabelsAwareModule, StarlarkValue { + + // Restrict for label ids like 'BAZEL_REV_ID'. More strict than our current revId. + private static final Pattern CUSTOM_REVID_FORMAT = Pattern.compile("[A-Z][A-Z_0-9]{1,30}_REV_ID"); + private static final String CHECK_LAST_REV_STATE = "check_last_rev_state"; + private final GeneralOptions generalOptions; + private final WorkflowOptions workflowOptions; + private final DebugOptions debugOptions; + private ConfigFile mainConfigFile; + private Supplier> allConfigFiles; + private Supplier dynamicStarlarkThread; + + public Core( + GeneralOptions generalOptions, WorkflowOptions workflowOptions, DebugOptions debugOptions) { + this.generalOptions = Preconditions.checkNotNull(generalOptions); + this.workflowOptions = Preconditions.checkNotNull(workflowOptions); + this.debugOptions = Preconditions.checkNotNull(debugOptions); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "reverse", + doc = + "Given a list of transformations, returns the list of transformations equivalent to" + + " undoing all the transformations", + parameters = { + @Param( + name = "transformations", + named = true, + type = com.google.devtools.build.lib.syntax.Sequence.class, + generic1 = Transformation.class, + doc = "The transformations to reverse"), + }) + public com.google.devtools.build.lib.syntax.Sequence reverse( + com.google.devtools.build.lib.syntax.Sequence + transforms // or + ) throws EvalException { + + ImmutableList.Builder builder = ImmutableList.builder(); + for (Object t : transforms) { + try { + if (t instanceof StarlarkCallable) { + builder.add( + new SkylarkTransformation((StarlarkCallable) t, Dict.empty(), dynamicStarlarkThread) + .reverse()); + } else if (t instanceof Transformation) { + builder.add(((Transformation) t).reverse()); + } else { + throw Starlark.errorf("Expected type 'transformation' or function, but found: %s", t); + } + } catch (NonReversibleValidationException e) { + throw Starlark.errorf("%s", e.getMessage()); + } + } + + return com.google.devtools.build.lib.syntax.StarlarkList.immutableCopyOf( + builder.build().reverse()); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "workflow", + doc = + "Defines a migration pipeline which can be invoked via the Copybara command.\n" + + "\n" + + "Implicit labels that can be used/exposed:\n" + + "\n" + + " - " + + TransformWork.COPYBARA_CONTEXT_REFERENCE_LABEL + + ": Requested reference. For example if copybara is invoked as `copybara" + + " copy.bara.sky workflow master`, the value would be `master`.\n" + + " - " + + TransformWork.COPYBARA_LAST_REV + + ": Last reference that was migrated\n" + + " - " + + TransformWork.COPYBARA_CURRENT_REV + + ": The current reference being migrated\n" + + " - " + + TransformWork.COPYBARA_CURRENT_MESSAGE + + ": The current message at this point of the transformations\n" + + " - " + + TransformWork.COPYBARA_CURRENT_MESSAGE_TITLE + + ": The current message title (first line) at this point of the transformations\n" + + " - " + + TransformWork.COPYBARA_AUTHOR + + ": The author of the change\n", + parameters = { + @Param( + name = "name", + named = true, + type = String.class, + doc = "The name of the workflow.", + positional = false), + @Param( + name = "origin", + named = true, + type = Origin.class, + doc = + "Where to read from the code to be migrated, before applying the " + + "transformations. This is usually a VCS like Git, but can also be a local " + + "folder or even a pending change in a code review system like Gerrit.", + positional = false), + @Param( + name = "destination", + named = true, + type = Destination.class, + doc = + "Where to write to the code being migrated, after applying the " + + "transformations. This is usually a VCS like Git, but can also be a local " + + "folder or even a pending change in a code review system like Gerrit.", + positional = false), + @Param( + name = "authoring", + named = true, + type = Authoring.class, + doc = "The author mapping configuration from origin to destination.", + positional = false), + @Param( + name = "transformations", + named = true, + type = com.google.devtools.build.lib.syntax.Sequence.class, + doc = "The transformations to be run for this workflow. They will run in sequence.", + positional = false, + defaultValue = "[]"), + @Param( + name = "origin_files", + named = true, + type = Glob.class, + doc = + "A glob relative to the workdir that will be read from the" + + " origin during the import. For example glob([\"**.java\"]), all java files," + + " recursively, which excludes all other file types.", + defaultValue = "None", + noneable = true, + positional = false), + @Param( + name = "destination_files", + named = true, + type = Glob.class, + doc = + "A glob relative to the root of the destination repository that matches files that" + + " are part of the migration. Files NOT matching this glob will never be" + + " removed, even if the file does not exist in the source. For example" + + " glob(['**'], exclude = ['**/BUILD']) keeps all BUILD files in destination" + + " when the origin does not have any BUILD files. You can also use this to" + + " limit the migration to a subdirectory of the destination, e.g." + + " glob(['java/src/**'], exclude = ['**/BUILD']) to only affect non-BUILD" + + " files in java/src.", + defaultValue = "None", + noneable = true, + positional = false), + @Param( + name = "mode", + named = true, + type = String.class, + doc = + "Workflow mode. Currently we support four modes:
  • 'SQUASH':" + + " Create a single commit in the destination with new tree" + + " state.
  • 'ITERATIVE': Import each origin change" + + " individually.
  • 'CHANGE_REQUEST': Import a pending change to" + + " the Source-of-Truth. This could be a GH Pull Request, a Gerrit Change," + + " etc. The final intention should be to submit the change in the SoT" + + " (destination in this case).
  • 'CHANGE_REQUEST_FROM_SOT':" + + " Import a pending change **from** the Source-of-Truth. This mode is useful" + + " when, despite the pending change being already in the SoT, the users want" + + " to review the code on a different system. The final intention should never" + + " be to submit in the destination, but just review or test
", + defaultValue = "\"SQUASH\"", + positional = false), + @Param( + name = "reversible_check", + named = true, + type = Boolean.class, + doc = + "Indicates if the tool should try to to reverse all the transformations" + + " at the end to check that they are reversible.
The default value is" + + " True for 'CHANGE_REQUEST' mode. False otherwise", + defaultValue = "None", + noneable = true, + positional = false), + @Param( + name = CHECK_LAST_REV_STATE, + named = true, + type = Boolean.class, + doc = + "If set to true, Copybara will validate that the destination didn't change" + + " since last-rev import for destination_files. Note that this" + + " flag doesn't work for CHANGE_REQUEST mode.", + defaultValue = "None", + noneable = true, + positional = false), + @Param( + name = "ask_for_confirmation", + named = true, + type = Boolean.class, + doc = + "Indicates that the tool should show the diff and require user's" + + " confirmation before making a change in the destination.", + defaultValue = "False", + positional = false), + @Param( + name = "dry_run", + named = true, + type = Boolean.class, + doc = + "Run the migration in dry-run mode. Some destination implementations might" + + " have some side effects (like creating a code review), but never submit to a" + + " main branch.", + defaultValue = "False", + positional = false), + @Param( + name = "after_migration", + named = true, + type = com.google.devtools.build.lib.syntax.Sequence.class, + doc = + "Run a feedback workflow after one migration happens. This runs once per" + + " change in `ITERATIVE` mode and only once for `SQUASH`.", + defaultValue = "[]", + positional = false), + @Param( + name = "after_workflow", + named = true, + type = com.google.devtools.build.lib.syntax.Sequence.class, + doc = + "Run a feedback workflow after all the changes for this workflow run are migrated." + + " Prefer `after_migration` as it is executed per change (in ITERATIVE mode)." + + " Tasks in this hook shouldn't be critical to execute. These actions" + + " shouldn't record effects (They'll be ignored).", + defaultValue = "[]", + positional = false), + @Param( + name = "change_identity", + named = true, + type = String.class, + doc = + "By default, Copybara hashes several fields so that each change has an unique" + + " identifier that at the same time reuses the generated destination change." + + " This allows to customize the identity hash generation so that the same" + + " identity is used in several workflows. At least ${copybara_config_path}" + + " has to be present. Current user is added to the hash" + + " automatically.

Available variables:
    " + + "
  • ${copybara_config_path}: Main config file path
    • " + + "
  • ${copybara_workflow_name}: The name of the workflow being run
    • " + + "
  • ${copybara_reference}: The requested reference. In general Copybara" + + " tries its best to give a repetable reference. For example Gerrit change" + + " number or change-id or GitHub Pull Request number. If it cannot find a" + + " context reference it uses the resolved revision.
    • " + + "
  • ${label:label_name}: A label present for the current change. Exposed" + + " in the message or not.
If any of the labels cannot be found it" + + " defaults to the default identity (The effect would be no reuse of" + + " destination change between workflows)", + defaultValue = "None", + noneable = true, + positional = false), + @Param( + name = "set_rev_id", + named = true, + type = Boolean.class, + doc = + "Copybara adds labels like 'GitOrigin-RevId' in the destination in order to" + + " track what was the latest change imported. For `CHANGE_REQUEST` " + + "workflows it is not used and is purely informational. This field " + + "allows to disable it for that mode. Destinations might ignore the flag.", + defaultValue = "True", + positional = false), + @Param( + name = "smart_prune", + named = true, + type = Boolean.class, + doc = + "By default CHANGE_REQUEST workflows cannot restore scrubbed files. This flag does" + + " a best-effort approach in restoring the non-affected snippets. For now we" + + " only revert the non-affected files. This only works for CHANGE_REQUEST" + + " mode.", + defaultValue = "False", + positional = false), + @Param( + name = "migrate_noop_changes", + named = true, + type = Boolean.class, + doc = + "By default, Copybara tries to only migrate changes that affect origin_files or" + + " config files. This flag allows to include all the changes. Note that it" + + " might generate more empty changes errors. In `ITERATIVE` mode it might" + + " fail if some transformation is validating the message (Like has to contain" + + " 'PUBLIC' and the change doesn't contain it because it is internal).", + defaultValue = "False", + positional = false), + @Param( + name = "experimental_custom_rev_id", + named = true, + type = String.class, + doc = + "Use this label name instead of the one provided by the origin. This is subject" + + " to change and there is no guarantee.", + defaultValue = "None", + positional = false, + noneable = true), + @Param( + name = "description", + type = String.class, + named = true, + noneable = true, + positional = false, + doc = "A description of what this workflow achieves", + defaultValue = "None"), + @Param( + name = "checkout", + type = Boolean.class, + named = true, + positional = false, + doc = + "Allows disabling the checkout. The usage of this feature is rare. This could" + + " be used to update a file of your own repo when a dependant repo version" + + " changes and you are not interested on the files of the dependant repo, just" + + " the new version.", + defaultValue = "True"), + }, + useStarlarkThread = true) + @UsesFlags({WorkflowOptions.class}) + @DocDefault(field = "origin_files", value = "glob([\"**\"])") + @DocDefault(field = "destination_files", value = "glob([\"**\"])") + @DocDefault(field = CHECK_LAST_REV_STATE, value = "True for CHANGE_REQUEST") + @DocDefault(field = "reversible_check", value = "True for 'CHANGE_REQUEST' mode. False otherwise") + public void workflow( + String workflowName, + Origin origin, // + Destination destination, + Authoring authoring, + com.google.devtools.build.lib.syntax.Sequence transformations, + Object originFiles, + Object destinationFiles, + String modeStr, + Object reversibleCheckObj, + Object checkLastRevStateField, + Boolean askForConfirmation, + Boolean dryRunMode, + com.google.devtools.build.lib.syntax.Sequence afterMigrations, + com.google.devtools.build.lib.syntax.Sequence afterAllMigrations, + Object changeIdentityObj, + Boolean setRevId, + Boolean smartPrune, + Boolean migrateNoopChanges, + Object customRevIdField, + Object description, + Boolean checkout, + StarlarkThread thread) + throws EvalException { + WorkflowMode mode = stringToEnum("mode", modeStr, WorkflowMode.class); + + Sequence sequenceTransform = + Sequence.fromConfig( + generalOptions.profiler(), + workflowOptions.joinTransformations(), + transformations, + "transformations", + dynamicStarlarkThread, + debugOptions::transformWrapper); + Transformation reverseTransform = null; + if (!generalOptions.isDisableReversibleCheck() + && convertFromNoneable(reversibleCheckObj, mode == WorkflowMode.CHANGE_REQUEST)) { + try { + reverseTransform = sequenceTransform.reverse(); + } catch (NonReversibleValidationException e) { + throw Starlark.errorf("%s", e.getMessage()); + } + } + + ImmutableList changeIdentity = getChangeIdentity(changeIdentityObj); + + String customRevId = convertFromNoneable(customRevIdField, null); + check( + customRevId == null || CUSTOM_REVID_FORMAT.matches(customRevId), + "Invalid experimental_custom_rev_id format. Format: %s", + CUSTOM_REVID_FORMAT.pattern()); + + if (setRevId) { + check( + mode != WorkflowMode.CHANGE_REQUEST || customRevId == null, + "experimental_custom_rev_id is not allowed to be used in CHANGE_REQUEST mode if" + + " set_rev_id is set to true. experimental_custom_rev_id is used for looking" + + " for the baseline in the origin. No revId is stored in the destination."); + } else { + check( + mode == WorkflowMode.CHANGE_REQUEST, + "'set_rev_id = False' is only supported" + " for CHANGE_REQUEST mode."); + } + if (smartPrune) { + check( + mode == WorkflowMode.CHANGE_REQUEST, + "'smart_prune = True' is only supported" + " for CHANGE_REQUEST mode."); + } + + boolean checkLastRevState = convertFromNoneable(checkLastRevStateField, false); + if (checkLastRevState) { + check( + mode != WorkflowMode.CHANGE_REQUEST, + "%s is not compatible with %s", + CHECK_LAST_REV_STATE, + WorkflowMode.CHANGE_REQUEST); + } + + Authoring resolvedAuthoring = authoring; + Author defaultAuthorFlag = workflowOptions.getDefaultAuthorFlag(); + if (defaultAuthorFlag != null) { + resolvedAuthoring = new Authoring(defaultAuthorFlag, authoring.getMode(), + authoring.getWhitelist()); + } + + WorkflowMode effectiveMode = generalOptions.squash ? WorkflowMode.SQUASH : mode; + Workflow workflow = + new Workflow<>( + workflowName, + convertFromNoneable(description, null), + (Origin) origin, + destination, + resolvedAuthoring, + sequenceTransform, + workflowOptions.getLastRevision(), + workflowOptions.isInitHistory(), + generalOptions, + convertFromNoneable(originFiles, Glob.ALL_FILES), + convertFromNoneable(destinationFiles, Glob.ALL_FILES), + effectiveMode, + workflowOptions, + reverseTransform, + askForConfirmation, + mainConfigFile, + allConfigFiles, + dryRunMode, + checkLastRevState || workflowOptions.checkLastRevState, + convertFeedbackActions(afterMigrations, dynamicStarlarkThread), + convertFeedbackActions(afterAllMigrations, dynamicStarlarkThread), + changeIdentity, + setRevId, + smartPrune, + workflowOptions.migrateNoopChanges || migrateNoopChanges, + customRevId, + checkout); + Module module = Module.ofInnermostEnclosingStarlarkFunction(thread); + registerGlobalMigration(workflowName, workflow, module); + } + + private static ImmutableList getChangeIdentity(Object changeIdentityObj) + throws EvalException { + String changeIdentity = convertFromNoneable(changeIdentityObj, null); + + if (changeIdentity == null) { + return ImmutableList.of(); + } + ImmutableList result = new Parser().parse(changeIdentity); + boolean configVarFound = false; + for (Token token : result) { + if (token.getType() != TokenType.INTERPOLATION) { + continue; + } + if (token.getValue().equals(COPYBARA_CONFIG_PATH_IDENTITY_VAR)) { + configVarFound = true; + continue; + } + if (token.getValue().equals(Workflow.COPYBARA_WORKFLOW_NAME_IDENTITY_VAR) + || token.getValue().equals(Workflow.COPYBARA_REFERENCE_IDENTITY_VAR) + || token.getValue().startsWith(Workflow.COPYBARA_REFERENCE_LABEL_VAR)) { + continue; + } + throw Starlark.errorf("Unrecognized variable: %s", token.getValue()); + } + check(configVarFound, "${%s} variable is required", COPYBARA_CONFIG_PATH_IDENTITY_VAR); + return result; + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "move", + doc = "Moves files between directories and renames files", + parameters = { + @Param( + name = "before", + named = true, + type = String.class, + doc = + "The name of the file or directory before moving. If this is the empty string and" + + " 'after' is a directory, then all files in the workdir will be moved to the" + + " sub directory specified by 'after', maintaining the directory tree."), + @Param( + name = "after", + named = true, + type = String.class, + doc = + "The name of the file or directory after moving. If this is the empty string and" + + " 'before' is a directory, then all files in 'before' will be moved to the" + + " repo root, maintaining the directory tree inside 'before'."), + @Param( + name = "paths", + named = true, + type = Glob.class, + doc = + "A glob expression relative to 'before' if it represents a directory." + + " Only files matching the expression will be moved. For example," + + " glob([\"**.java\"]), matches all java files recursively inside" + + " 'before' folder. Defaults to match all the files recursively.", + defaultValue = "None", + noneable = true), + @Param( + name = "overwrite", + named = true, + doc = + "Overwrite destination files if they already exist. Note that this makes the" + + " transformation non-reversible, since there is no way to know if the file" + + " was overwritten or not in the reverse workflow.", + type = Boolean.class, + defaultValue = "False") + }, + useStarlarkThread = true) + @DocDefault(field = "paths", value = "glob([\"**\"])") + @Example( + title = "Move a directory", + before = "Move all the files in a directory to another directory:", + code = "core.move(\"foo/bar_internal\", \"bar\")", + after = "In this example, `foo/bar_internal/one` will be moved to `bar/one`.") + @Example( + title = "Move all the files to a subfolder", + before = "Move all the files in the checkout dir into a directory called foo:", + code = "core.move(\"\", \"foo\")", + after = "In this example, `one` and `two/bar` will be moved to `foo/one` and `foo/two/bar`.") + @Example( + title = "Move a subfolder's content to the root", + before = "Move the contents of a folder to the checkout root directory:", + code = "core.move(\"foo\", \"\")", + after = "In this example, `foo/bar` would be moved to `bar`.") + public Transformation move( + String before, String after, Object paths, Boolean overwrite, StarlarkThread thread) + throws EvalException { + + check( + !Objects.equals(before, after), + "Moving from the same folder to the same folder is a noop. Remove the" + + " transformation."); + + return CopyOrMove.createMove( + before, + after, + workflowOptions, + convertFromNoneable(paths, Glob.ALL_FILES), + overwrite, + thread.getCallerLocation()); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "copy", + doc = "Copy files between directories and renames files", + parameters = { + @Param( + name = "before", + named = true, + type = String.class, + doc = + "The name of the file or directory to copy. If this is the empty string and" + + " 'after' is a directory, then all files in the workdir will be copied to" + + " the sub directory specified by 'after', maintaining the directory tree."), + @Param( + name = "after", + named = true, + type = String.class, + doc = + "The name of the file or directory destination. If this is the empty string and" + + " 'before' is a directory, then all files in 'before' will be copied to the" + + " repo root, maintaining the directory tree inside 'before'."), + @Param( + name = "paths", + named = true, + type = Glob.class, + doc = + "A glob expression relative to 'before' if it represents a directory." + + " Only files matching the expression will be copied. For example," + + " glob([\"**.java\"]), matches all java files recursively inside" + + " 'before' folder. Defaults to match all the files recursively.", + defaultValue = "None", + noneable = true), + @Param( + name = "overwrite", + named = true, + doc = + "Overwrite destination files if they already exist. Note that this makes the" + + " transformation non-reversible, since there is no way to know if the file" + + " was overwritten or not in the reverse workflow.", + type = Boolean.class, + defaultValue = "False") + }, + useStarlarkThread = true) + @DocDefault(field = "paths", value = "glob([\"**\"])") + @Example( + title = "Copy a directory", + before = "Move all the files in a directory to another directory:", + code = "core.copy(\"foo/bar_internal\", \"bar\")", + after = "In this example, `foo/bar_internal/one` will be copied to `bar/one`.") + @Example( + title = "Copy with reversal", + before = "Copy all static files to a 'static' folder and use remove for reverting the change", + code = + "core.transform(\n" + + " [core.copy(\"foo\", \"foo/static\", paths = glob([\"**.css\",\"**.html\"," + + " ]))],\n" + + " reversal = [core.remove(glob(['foo/static/**.css'," + + " 'foo/static/**.html']))]\n" + + ")") + public Transformation copy( + String before, String after, Object paths, Boolean overwrite, StarlarkThread thread) + throws EvalException { + check( + !Objects.equals(before, after), + "Copying from the same folder to the same folder is a noop. Remove the" + + " transformation."); + return CopyOrMove.createCopy( + before, + after, + workflowOptions, + convertFromNoneable(paths, Glob.ALL_FILES), + overwrite, + thread.getCallerLocation()); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "remove", + doc = + "Remove files from the workdir. **This transformation is only meant to be used inside" + + " core.transform for reversing core.copy like transforms**. For regular file" + + " filtering use origin_files exclude mechanism.", + parameters = { + @Param(name = "paths", named = true, type = Glob.class, doc = "The files to be deleted"), + }, + useStarlarkThread = true) + @Example( + title = "Reverse a file copy", + before = "Move all the files in a directory to another directory:", + code = + "core.transform(\n" + + " [core.copy(\"foo\", \"foo/public\")],\n" + + " reversal = [core.remove(glob([\"foo/public/**\"]))])", + after = "In this example, `foo/one` will be moved to `foo/public/one`.") + @Example( + title = "Copy with reversal", + before = "Copy all static files to a 'static' folder and use remove for reverting the change", + code = + "core.transform(\n" + + " [core.copy(\"foo\", \"foo/static\", paths = glob([\"**.css\",\"**.html\"," + + " ]))],\n" + + " reversal = [core.remove(glob(['foo/static/**.css'," + + " 'foo/static/**.html']))]\n" + + ")") + public Remove remove(Glob paths, StarlarkThread thread) { + return new Remove(paths, workflowOptions, thread.getCallerLocation()); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "replace", + doc = + "Replace a text with another text using optional regex groups. This tranformer can be" + + " automatically reversed.", + parameters = { + @Param( + name = "before", + named = true, + type = String.class, + doc = + "The text before the transformation. Can contain references to regex groups. For" + + " example \"foo${x}text\".

`before` can only contain 1 reference to each" + + " unique `regex_group`. If you require multiple references to the same" + + " `regex_group`, add `repeated_groups: True`.

If '$' literal character" + + " needs to be matched, '`$$`' should be used. For example '`$$FOO`' would" + + " match the literal '$FOO'."), + @Param( + name = "after", + named = true, + type = String.class, + doc = + "The text after the transformation. It can also contain references to regex " + + "groups, like 'before' field."), + @Param( + name = "regex_groups", + named = true, + type = Dict.class, + doc = + "A set of named regexes that can be used to match part of the replaced text." + + "Copybara uses [re2](https://github.com/google/re2/wiki/Syntax) syntax." + + " For example {\"x\": \"[A-Za-z]+\"}", + defaultValue = "{}"), + @Param( + name = "paths", + named = true, + type = Glob.class, + doc = + "A glob expression relative to the workdir representing the files to apply the" + + " transformation. For example, glob([\"**.java\"]), matches all java files" + + " recursively. Defaults to match all the files recursively.", + defaultValue = "None", + noneable = true), + @Param( + name = "first_only", + named = true, + type = Boolean.class, + doc = + "If true, only replaces the first instance rather than all. In single line mode," + + " replaces the first instance on each line. In multiline mode, replaces the" + + " first instance in each file.", + defaultValue = "False"), + @Param( + name = "multiline", + named = true, + type = Boolean.class, + doc = "Whether to replace text that spans more than one line.", + defaultValue = "False"), + @Param( + name = "repeated_groups", + named = true, + type = Boolean.class, + doc = + "Allow to use a group multiple times. For example foo${repeated}/${repeated}. Note" + + " that this mechanism doesn't use backtracking. In other words, the group" + + " instances are treated as different groups in regex construction and then a" + + " validation is done after that.", + defaultValue = "False"), + @Param( + name = "ignore", + named = true, + type = com.google.devtools.build.lib.syntax.Sequence.class, + doc = + "A set of regexes. Any text that matches any expression in this set, which" + + " might otherwise be transformed, will be ignored.", + defaultValue = "[]"), + }, + useStarlarkThread = true) + @DocDefault(field = "paths", value = "glob([\"**\"])") + @Example( + title = "Simple replacement", + before = "Replaces the text \"internal\" with \"external\" in all java files", + code = + "core.replace(\n" + + " before = \"internal\",\n" + + " after = \"external\",\n" + + " paths = glob([\"**.java\"]),\n" + + ")") + @Example( + title = "Append some text at the end of files", + before = "", + code = + "core.replace(\n" + + " before = '${end}',\n" + + " after = 'Text to be added at the end',\n" + + " multiline = True,\n" + + " regex_groups = { 'end' : '\\z'},\n" + + ")") + @Example( + title = "Append some text at the end of files reversible", + before = "Same as the above example but make the transformation reversible", + code = + "core.transform([\n" + + " core.replace(\n" + + " before = '${end}',\n" + + " after = 'some append',\n" + + " multiline = True,\n" + + " regex_groups = { 'end' : '\\z'},\n" + + " )\n" + + "],\n" + + "reversal = [\n" + + " core.replace(\n" + + " before = 'some append${end}',\n" + + " after = '',\n" + + " multiline = True,\n" + + " regex_groups = { 'end' : '\\z'},\n" + + " )" + + "])") + @Example( + title = "Replace using regex groups", + before = + "In this example we map some urls from the internal to the external version in" + + " all the files of the project.", + code = + "core.replace(\n" + + " before = \"https://some_internal/url/${pkg}.html\",\n" + + " after = \"https://example.com/${pkg}.html\",\n" + + " regex_groups = {\n" + + " \"pkg\": \".*\",\n" + + " },\n" + + " )", + after = + "So a url like `https://some_internal/url/foo/bar.html` will be transformed to" + + " `https://example.com/foo/bar.html`.") + @Example( + title = "Remove confidential blocks", + before = + "This example removes blocks of text/code that are confidential and thus shouldn't" + + "be exported to a public repository.", + code = + "core.replace(\n" + + " before = \"${x}\",\n" + + " after = \"\",\n" + + " multiline = True,\n" + + " regex_groups = {\n" + + " \"x\": \"(?m)^.*BEGIN-INTERNAL[\\\\w\\\\W]*?END-INTERNAL.*$\\\\n\",\n" + + " },\n" + + " )", + after = + "This replace would transform a text file like:\n\n" + + "```\n" + + "This is\n" + + "public\n" + + " // BEGIN-INTERNAL\n" + + " confidential\n" + + " information\n" + + " // END-INTERNAL\n" + + "more public code\n" + + " // BEGIN-INTERNAL\n" + + " more confidential\n" + + " information\n" + + " // END-INTERNAL\n" + + "```\n\n" + + "Into:\n\n" + + "```\n" + + "This is\n" + + "public\n" + + "more public code\n" + + "```\n\n") + public Replace replace( + String before, + String after, + Dict regexes, // + Object paths, + Boolean firstOnly, + Boolean multiline, + Boolean repeatedGroups, + com.google.devtools.build.lib.syntax.Sequence ignore, // + StarlarkThread thread) + throws EvalException { + return Replace.create( + thread.getCallerLocation(), + before, + after, + SkylarkUtil.convertStringMap(regexes, "regex_groups"), + convertFromNoneable(paths, Glob.ALL_FILES), + firstOnly, + multiline, + repeatedGroups, + SkylarkUtil.convertStringList(ignore, "patterns_to_ignore"), + workflowOptions); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "todo_replace", + doc = "Replace Google style TODOs. For example `TODO(username, othername)`.", + parameters = { + @Param( + name = "tags", + named = true, + type = com.google.devtools.build.lib.syntax.Sequence.class, + generic1 = String.class, + doc = "Prefix tag to look for", + defaultValue = "['TODO', 'NOTE']"), + @Param( + name = "mapping", + named = true, + type = Dict.class, + doc = "Mapping of users/strings", + defaultValue = "{}"), + @Param( + name = "mode", + named = true, + type = String.class, + doc = + "Mode for the replace:

  • 'MAP_OR_FAIL': Try to use the mapping and if not" + + " found fail.
  • 'MAP_OR_IGNORE': Try to use the mapping but ignore if" + + " no mapping found.
  • 'MAP_OR_DEFAULT': Try to use the mapping and use" + + " the default if not found.
  • 'SCRUB_NAMES': Scrub all names from" + + " TODOs. Transforms 'TODO(foo)' to 'TODO'
  • 'USE_DEFAULT': Replace any" + + " TODO(foo, bar) with TODO(default_string)
", + defaultValue = "'MAP_OR_IGNORE'"), + @Param( + name = "paths", + named = true, + type = Glob.class, + doc = + "A glob expression relative to the workdir representing the files to apply the" + + " transformation. For example, glob([\"**.java\"]), matches all java files" + + " recursively. Defaults to match all the files recursively.", + defaultValue = "None", + noneable = true), + @Param( + name = "default", + named = true, + type = String.class, + doc = + "Default value if mapping not found. Only valid for 'MAP_OR_DEFAULT' or" + + " 'USE_DEFAULT' modes", + noneable = true, + defaultValue = "None"), + @Param( + name = "ignore", + named = true, + type = String.class, + doc = + "If set, elements within TODO (with usernames) that match the regex will be " + + "ignored. For example ignore = \"foo\" would ignore \"foo\" in " + + "\"TODO(foo,bar)\" but not \"bar\".", + defaultValue = "None", + noneable = true), + }, + useStarlarkThread = true) + @DocDefault(field = "paths", value = "glob([\"**\"])") + @Example( + title = "Simple update", + before = "Replace TODOs and NOTES for users in the mapping:", + code = + "core.todo_replace(\n" + + " mapping = {\n" + + " 'test1' : 'external1',\n" + + " 'test2' : 'external2'\n" + + " }\n" + + ")", + after = + "Would replace texts like TODO(test1) or NOTE(test1, test2) with TODO(external1)" + + " or NOTE(external1, external2)") + @Example( + title = "Scrubbing", + before = "Remove text from inside TODOs", + code = "core.todo_replace(\n" + " mode = 'SCRUB_NAMES'\n" + ")", + after = + "Would replace texts like TODO(test1): foo or NOTE(test1, test2):foo with TODO:foo" + + " and NOTE:foo") + @Example( + title = "Ignoring Regex Patterns", + before = "Ignore regEx inside TODOs when scrubbing/mapping", + code = "core.todo_replace(\n" + " mapping = { 'aaa' : 'foo'},\n" + " ignore = 'b/.*'\n)", + after = "Would replace texts like TODO(b/123, aaa) with TODO(b/123, foo)") + public TodoReplace todoReplace( + com.google.devtools.build.lib.syntax.Sequence skyTags, // + Dict skyMapping, // + String modeStr, + Object paths, + Object skyDefault, + Object regexToIgnore, + StarlarkThread thread) + throws EvalException { + Mode mode = stringToEnum("mode", modeStr, Mode.class); + Map mapping = SkylarkUtil.convertStringMap(skyMapping, "mapping"); + String defaultString = convertFromNoneable(skyDefault, /*defaultValue=*/null); + ImmutableList tags = + ImmutableList.copyOf(SkylarkUtil.convertStringList(skyTags, "tags")); + String ignorePattern = convertFromNoneable(regexToIgnore, null); + Pattern regexWhitelist = ignorePattern != null ? Pattern.compile(ignorePattern) : null; + + check(!tags.isEmpty(), "'tags' cannot be empty"); + if (mode == Mode.MAP_OR_DEFAULT || mode == Mode.USE_DEFAULT) { + check(defaultString != null, "'default' needs to be set for mode '%s'", mode); + } else { + check(defaultString == null, "'default' cannot be used for mode '%s'", mode); + } + if (mode == Mode.USE_DEFAULT || mode == Mode.SCRUB_NAMES) { + check(mapping.isEmpty(), "'mapping' cannot be used with mode %s", mode); + } + return new TodoReplace( + thread.getCallerLocation(), + convertFromNoneable(paths, Glob.ALL_FILES), + tags, + mode, + mapping, + defaultString, + workflowOptions.parallelizer(), + regexWhitelist); + } + + public static final String TODO_FILTER_REPLACE_EXAMPLE = "" + + "core.filter_replace(\n" + + " regex = 'TODO\\((.*?)\\)',\n" + + " group = 1,\n" + + " mapping = core.replace_mapper([\n" + + " core.replace(\n" + + " before = '${p}foo${s}',\n" + + " after = '${p}fooz${s}',\n" + + " regex_groups = { 'p': '.*', 's': '.*'}\n" + + " ),\n" + + " core.replace(\n" + + " before = '${p}baz${s}',\n" + + " after = '${p}bazz${s}',\n" + + " regex_groups = { 'p': '.*', 's': '.*'}\n" + + " ),\n" + + " ],\n" + + " all = True\n" + + " )\n" + + ")"; + + public static final String SIMPLE_FILTER_REPLACE_EXAMPLE = "" + + "core.filter_replace(\n" + + " regex = 'a.*',\n" + + " mapping = {\n" + + " 'afoo': 'abar',\n" + + " 'abaz': 'abam'\n" + + " }\n" + + ")"; + + @SuppressWarnings({"unused", "unchecked"}) + @StarlarkMethod( + name = "filter_replace", + doc = + "Applies an initial filtering to find a substring to be replaced and then applies" + + " a `mapping` of replaces for the matched text.", + parameters = { + @Param( + name = "regex", + named = true, + type = String.class, + doc = "A re2 regex to match a substring of the file"), + @Param( + name = "mapping", + named = true, + doc = "A mapping function like core.replace_mapper or a dict with mapping values.", + defaultValue = "{}"), + @Param( + name = "group", + named = true, + type = Integer.class, + doc = + "Extract a regex group from the matching text and pass this as parameter to" + + " the mapping instead of the whole matching text.", + noneable = true, + defaultValue = "None"), + @Param( + name = "paths", + named = true, + type = Glob.class, + doc = + "A glob expression relative to the workdir representing the files to apply the" + + " transformation. For example, glob([\"**.java\"]), matches all java files" + + " recursively. Defaults to match all the files recursively.", + defaultValue = "None", + noneable = true), + @Param( + name = "reverse", + named = true, + type = String.class, + doc = "A re2 regex used as reverse transformation", + defaultValue = "None", + noneable = true), + }, + useStarlarkThread = true) + @DocDefault(field = "paths", value = "glob([\"**\"])") + @DocDefault(field = "reverse", value = "`regex`") + @DocDefault(field = "group", value = "Whole text") + @Example( + title = "Simple replace with mapping", + before = "Simplest mapping", + code = SIMPLE_FILTER_REPLACE_EXAMPLE) + @Example( + title = "TODO replace", + before = "This replace is similar to what it can be achieved with core.todo_replace:", + code = TODO_FILTER_REPLACE_EXAMPLE) + public FilterReplace filterReplace( + String regex, + Object mapping, + Object group, + Object paths, + Object reverse, + StarlarkThread thread) + throws EvalException { + ReversibleFunction func = + getMappingFunction(mapping, thread.getCallerLocation()); + + String afterPattern = convertFromNoneable(reverse, regex); + int numGroup = convertFromNoneable(group, 0); + Pattern before = Pattern.compile(regex); + check( + numGroup <= before.groupCount(), + "group idx is greater than the number of groups defined in '%s'. Regex has %s groups", + before.pattern(), + before.groupCount()); + Pattern after = Pattern.compile(afterPattern); + check( + numGroup <= after.groupCount(), + "reverse_group idx is greater than the number of groups defined in '%s'." + + " Regex has %s groups", + after.pattern(), + after.groupCount()); + return new FilterReplace( + workflowOptions, + before, + after, + numGroup, + numGroup, + func, + convertFromNoneable(paths, Glob.ALL_FILES), + thread.getCallerLocation()); + } + + @SuppressWarnings("unchecked") + public static ReversibleFunction getMappingFunction(Object mapping, + Location location) + throws EvalException { + if (mapping instanceof Dict) { + ImmutableMap map = + ImmutableMap.copyOf(Dict.noneableCast(mapping, String.class, String.class, "mapping")); + check(!map.isEmpty(), "Empty mapping is not allowed." + " Remove the transformation instead"); + return new MapMapper(map, location); + } + check( + mapping instanceof ReversibleFunction, + "mapping has to be instance of" + " map or a reversible function"); + return (ReversibleFunction) mapping; + } + + @StarlarkMethod( + name = "replace_mapper", + doc = + "A mapping function that applies a list of replaces until one replaces the text" + + " (Unless `all = True` is used). This should be used with core.filter_replace or" + + " other transformations that accept text mapping as parameter.", + parameters = { + @Param( + name = "mapping", + type = com.google.devtools.build.lib.syntax.Sequence.class, + generic1 = Transformation.class, + named = true, + doc = "The list of core.replace transformations"), + @Param( + name = "all", + type = Boolean.class, + named = true, + positional = false, + doc = "Run all the mappings despite a replace happens.", + defaultValue = "False"), + }) + public ReplaceMapper mapImports( + com.google.devtools.build.lib.syntax.Sequence mapping, // + Boolean all) + throws EvalException { + check(!mapping.isEmpty(), "Empty mapping is not allowed"); + ImmutableList.Builder replaces = ImmutableList.builder(); + for (Transformation t : + com.google.devtools.build.lib.syntax.Sequence.cast( + mapping, Transformation.class, "mapping")) { + check( + t instanceof Replace, + "Only core.replace can be used as mapping, but got: " + t.describe()); + Replace replace = (Replace) t; + check( + replace.getPaths().equals(Glob.ALL_FILES), + "core.replace cannot use" + " 'paths' inside core.replace_mapper"); + replaces.add(replace); + } + return new ReplaceMapper(replaces.build(), all); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "verify_match", + doc = + "Verifies that a RegEx matches (or not matches) the specified files. Does not" + + " transform anything, but will stop the workflow if it fails.", + parameters = { + @Param( + name = "regex", + named = true, + type = String.class, + doc = + "The regex pattern to verify. To satisfy the validation, there has to be at" + + "least one (or no matches if verify_no_match) match in each of the files " + + "included in paths. The re2j pattern will be applied in multiline mode, i.e." + + " '^' refers to the beginning of a file and '$' to its end. " + + "Copybara uses [re2](https://github.com/google/re2/wiki/Syntax) syntax."), + @Param( + name = "paths", + named = true, + type = Glob.class, + doc = + "A glob expression relative to the workdir representing the files to apply the" + + " transformation. For example, glob([\"**.java\"]), matches all java files" + + " recursively. Defaults to match all the files recursively.", + defaultValue = "None", + noneable = true), + @Param( + name = "verify_no_match", + named = true, + type = Boolean.class, + doc = "If true, the transformation will verify that the RegEx does not match.", + defaultValue = "False"), + @Param( + name = "also_on_reversal", + named = true, + type = Boolean.class, + doc = + "If true, the check will also apply on the reversal. The default behavior is to" + + " not verify the pattern on reversal.", + defaultValue = "False"), + }, + useStarlarkThread = true) + @DocDefault(field = "paths", value = "glob([\"**\"])") + public VerifyMatch verifyMatch( + String regex, + Object paths, + Boolean verifyNoMatch, + Boolean alsoOnReversal, + StarlarkThread thread) + throws EvalException { + return VerifyMatch.create( + thread.getCallerLocation(), + regex, + convertFromNoneable(paths, Glob.ALL_FILES), + verifyNoMatch, + alsoOnReversal, + workflowOptions.parallelizer()); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "transform", + doc = + "Groups some transformations in a transformation that can contain a particular," + + " manually-specified, reversal, where the forward version and reversed version" + + " of the transform are represented as lists of transforms. The is useful if a" + + " transformation does not automatically reverse, or if the automatic reversal" + + " does not work for some reason." + + "
" + + "If reversal is not provided, the transform will try to compute the reverse of" + + " the transformations list.", + parameters = { + @Param( + name = "transformations", + type = com.google.devtools.build.lib.syntax.Sequence.class, + generic1 = Transformation.class, + named = true, + doc = + "The list of transformations to run as a result of running this" + + " transformation."), + @Param( + name = "reversal", + type = com.google.devtools.build.lib.syntax.Sequence.class, + generic1 = Transformation.class, + doc = + "The list of transformations to run as a result of running this" + + " transformation in reverse.", + named = true, + positional = false, + noneable = true, + defaultValue = "None"), + @Param( + name = "ignore_noop", + type = Boolean.class, + doc = + "In case a noop error happens in the group of transformations (Both forward and" + + " reverse), it will be ignored, but the rest of the transformations in the" + + " group will still be executed. If ignore_noop is not set," + + " we will apply the closest parent's ignore_noop.", + named = true, + positional = false, + noneable = true, + defaultValue = "None") + }, + useStarlarkThread = true) + @DocDefault(field = "reversal", value = "The reverse of 'transformations'") + public Transformation transform( + com.google.devtools.build.lib.syntax.Sequence transformations, // + Object reversal, + Object ignoreNoop, + StarlarkThread thread) + throws EvalException { + Sequence forward = + Sequence.fromConfig( + generalOptions.profiler(), + workflowOptions.joinTransformations(), + transformations, + "transformations", + dynamicStarlarkThread, + debugOptions::transformWrapper); + com.google.devtools.build.lib.syntax.Sequence reverseList = + convertFromNoneable(reversal, null); + Boolean updatedIgnoreNoop = convertFromNoneable(ignoreNoop, null); + if (reverseList == null) { + try { + reverseList = + com.google.devtools.build.lib.syntax.StarlarkList.immutableCopyOf( + ImmutableList.of(forward.reverse())); + } catch (NonReversibleValidationException e) { + throw Starlark.errorf( + "transformations are not automatically reversible." + + " Use 'reversal' field to explicitly configure the reversal of the transform"); + } + } + Sequence reverse = + Sequence.fromConfig( + generalOptions.profiler(), + workflowOptions.joinTransformations(), + reverseList, + "reversal", + dynamicStarlarkThread, + debugOptions::transformWrapper); + return new ExplicitReversal( + forward, reverse, updatedIgnoreNoop); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "dynamic_transform", + doc = + "Create a dynamic Skylark transformation. This should only be used by libraries" + + " developers", + parameters = { + @Param( + name = "impl", + named = true, + type = StarlarkCallable.class, + doc = "The Skylark function to call"), + @Param( + name = "params", + named = true, + type = Dict.class, + doc = "The parameters to the function. Will be available under ctx.params", + defaultValue = "{}"), + }, + useStarlarkThread = true) + @Example( + title = "Create a dynamic transformation with parameter", + before = + "If you want to create a library that uses dynamic transformations, you probably want to" + + " make them customizable. In order to do that, in your library.bara.sky, you need" + + " to hide the dynamic transformation (prefix with '_' and instead expose a" + + " function that creates the dynamic transformation with the param:", + code = + "" + + "def _test_impl(ctx):\n" + + " ctx.set_message(" + + "ctx.message + ctx.params['name'] + str(ctx.params['number']) + '\\n')\n" + + "\n" + + "def test(name, number = 2):\n" + + " return core.dynamic_transform(impl = _test_impl,\n" + + " params = { 'name': name, 'number': number})", + testExistingVariable = "test", + after = + "After defining this function, you can use `test('example', 42)` as a transformation" + + " in `core.workflow`.") + public Transformation dynamic_transform( + StarlarkCallable impl, Dict params, StarlarkThread thread) { + return new SkylarkTransformation( + impl, Dict.copyOf(thread.mutability(), params), dynamicStarlarkThread); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "dynamic_feedback", + doc = + "Create a dynamic Skylark feedback migration. This should only be used by libraries" + + " developers", + parameters = { + @Param( + name = "impl", + named = true, + type = StarlarkCallable.class, + doc = "The Skylark function to call"), + @Param( + name = "params", + named = true, + type = Dict.class, + doc = "The parameters to the function. Will be available under ctx.params", + defaultValue = "{}"), + }, + useStarlarkThread = true) + public Action dynamicFeedback(StarlarkCallable impl, Dict params, StarlarkThread thread) { + return new SkylarkAction( + impl, Dict.copyOf(thread.mutability(), params), dynamicStarlarkThread); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "fail_with_noop", + doc = "If invoked, it will fail the current migration as a noop", + parameters = { + @Param(name = "msg", named = true, type = String.class, doc = "The noop message"), + }, + useStarlarkThread = true) + public Action failWithNoop(String msg, StarlarkThread thread) throws EmptyChangeException { + // Add an internal EvalException to know the location of the error. + throw new EmptyChangeException(new EvalException(thread.getCallerLocation(), msg), msg); + } + + @StarlarkMethod(name = "main_config_path", + doc = "Location of the config file. This is subject to change", + structField = true) + public String getMainConfigFile() { + return mainConfigFile.getIdentifier(); + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "feedback", + doc = + "Defines a migration of changes' metadata, that can be invoked via the Copybara command" + + " in the same way as a regular workflow migrates the change itself.\n" + + "\n" + + "It is considered change metadata any information associated with a change" + + " (pending or submitted) that is not core to the change itself. A few examples:\n" + + "
    \n" + + "
  • Comments: Present in any code review system. Examples: Github PRs or" + + " Gerrit code reviews.
    • \n" + + "
  • Labels: Used in code review systems for approvals and/or CI results. " + + " Examples: Github labels, Gerrit code review labels.
    • \n" + + "
\n" + + "For the purpose of this workflow, it is not considered metadata the commit" + + " message in Git, or any of the contents of the file tree.\n" + + "\n", + parameters = { + @Param( + name = "name", + type = String.class, + doc = "The name of the feedback workflow.", + positional = false, + named = true), + @Param( + name = "origin", + type = Trigger.class, + doc = "The trigger of a feedback migration.", + positional = false, + named = true), + @Param( + name = "destination", + type = EndpointProvider.class, + doc = + "Where to write change metadata to. This is usually a code review system like " + + "Gerrit or GitHub PR.", + positional = false, + named = true), + @Param( + name = "actions", + type = com.google.devtools.build.lib.syntax.Sequence.class, + doc = + "" + + "A list of feedback actions to perform, with the following semantics:\n" + + " - There is no guarantee of the order of execution.\n" + + " - Actions need to be independent from each other.\n" + + " - Failure in one action might prevent other actions from executing.\n", + defaultValue = "[]", + positional = false, + named = true), + @Param( + name = "description", + type = String.class, + named = true, + noneable = true, + positional = false, + doc = "A description of what this workflow achieves", + defaultValue = "None"), + }, + useStarlarkThread = true) + @UsesFlags({FeedbackOptions.class}) + /*TODO(danielromero): Add default values*/ + public NoneType feedback( + String workflowName, + Trigger trigger, + EndpointProvider destination, + com.google.devtools.build.lib.syntax.Sequence feedbackActions, + Object description, + StarlarkThread thread) + throws EvalException { + ImmutableList actions = convertFeedbackActions(feedbackActions, dynamicStarlarkThread); + Feedback migration = + new Feedback( + workflowName, + convertFromNoneable(description, null), + mainConfigFile, + trigger, + destination.getEndpoint(), + actions, + generalOptions); + Module module = Module.ofInnermostEnclosingStarlarkFunction(thread); + registerGlobalMigration(workflowName, migration, module); + return Starlark.NONE; + } + + /** Registers a {@link Migration} in the global registry. */ + protected void registerGlobalMigration(String name, Migration migration, Module module) + throws EvalException { + getGlobalMigrations(module).addMigration(name, migration); + } + + @StarlarkMethod( + name = "format", + doc = "Formats a String using Java format patterns.", + parameters = { + @Param(name = "format", type = String.class, named = true, doc = "The format string"), + @Param( + name = "args", + type = com.google.devtools.build.lib.syntax.Sequence.class, + named = true, + doc = "The arguments to format"), + }) + public String format(String format, com.google.devtools.build.lib.syntax.Sequence args) + throws EvalException { + try { + return String.format(format, args.toArray(new Object[0])); + } catch (IllegalFormatException e) { + throw Starlark.errorf("Invalid format: %s: %s", format, e.getMessage()); + } + } + + private static ImmutableList convertFeedbackActions( + com.google.devtools.build.lib.syntax.Sequence feedbackActions, + Supplier thread) + throws EvalException { + ImmutableList.Builder actions = ImmutableList.builder(); + for (Object action : feedbackActions) { + if (action instanceof StarlarkCallable) { + actions.add(new SkylarkAction((StarlarkCallable) action, Dict.empty(), thread)); + } else if (action instanceof Action) { + actions.add((Action) action); + } else { + throw Starlark.errorf( + "Invalid feedback action '%s 'of type: %s", action, action.getClass()); + } + } + return actions.build(); + } + + @Override + public void setConfigFile(ConfigFile mainConfigFile, ConfigFile currentConfigFile) { + this.mainConfigFile = mainConfigFile; + } + + @Override + public void setAllConfigResources( + Supplier> allConfigFiles) { + this.allConfigFiles = allConfigFiles; + } + + @Override + public void setDynamicEnvironment(Supplier dynamicStarlarkThread) { + this.dynamicStarlarkThread = dynamicStarlarkThread; + } +} diff --git a/third_party/copybara/java/com/google/copybara/CoreGlobal.java b/third_party/copybara/java/com/google/copybara/CoreGlobal.java new file mode 100644 index 0000000000..1523b84105 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/CoreGlobal.java @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.copybara.authoring.Author; +import com.google.copybara.config.SkylarkUtil; +import com.google.copybara.doc.annotations.Example; +import com.google.copybara.util.Glob; +import com.google.devtools.build.lib.skylarkinterface.Param; +import com.google.devtools.build.lib.skylarkinterface.StarlarkGlobalLibrary; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Sequence; +import com.google.devtools.build.lib.syntax.Starlark; +import com.google.devtools.build.lib.syntax.StarlarkValue; +import java.util.List; + +/** + * A module to expose Skylark glob(), parse_message(), etc functions. + * + *

Don't add functions here and prefer "core" namespace unless it is something really general + */ +@StarlarkGlobalLibrary +public class CoreGlobal implements StarlarkValue { + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "glob", + doc = + "Glob returns a list of every file in the workdir that matches at least one" + + " pattern in include and does not match any of the patterns in exclude.", + parameters = { + @Param( + name = "include", + type = Sequence.class, + named = true, + generic1 = String.class, + doc = "The list of glob patterns to include"), + @Param( + name = "exclude", + type = Sequence.class, + generic1 = String.class, + doc = "The list of glob patterns to exclude", + defaultValue = "[]", + named = true, + positional = false), + }) + @Example( + title = "Simple usage", + before = "Include all the files under a folder except for `internal` folder files:", + code = "glob([\"foo/**\"], exclude = [\"foo/internal/**\"])") + @Example( + title = "Multiple folders", + before = "Globs can have multiple inclusive rules:", + code = "glob([\"foo/**\", \"bar/**\", \"baz/**.java\"])", + after = + "This will include all files inside `foo` and `bar` folders and Java files" + + " inside `baz` folder.") + @Example( + title = "Multiple excludes", + before = "Globs can have multiple exclusive rules:", + code = "glob([\"foo/**\"], exclude = [\"foo/internal/**\", \"foo/confidential/**\" ])", + after = + "Include all the files of `foo` except the ones in `internal` and `confidential`" + + " folders") + @Example( + title = "All BUILD files recursively", + before = + "Copybara uses Java globbing. The globbing is very similar to Bash one. This" + + " means that recursive globbing for a filename is a bit more tricky:", + code = "glob([\"BUILD\", \"**/BUILD\"])", + after = + "This is the correct way of matching all `BUILD` files recursively, including the" + + " one in the root. `**/BUILD` would only match `BUILD` files in subdirectories.") + @Example( + title = "Matching multiple strings with one expression", + before = + "While two globs can be used for matching two directories, there is a more" + + " compact approach:", + code = "glob([\"{java,javatests}/**\"])", + after = "This matches any file in `java` and `javatests` folders.") + @Example( + title = "Glob union", + before = + "This is useful when you want to exclude a broad subset of files but you want to" + + " still include some of those files.", + code = + "glob([\"folder/**\"], exclude = [\"folder/**.excluded\"])" + + " + glob([\'folder/includeme.excluded\'])", + after = + "This matches all the files in `folder`, excludes all files in that folder that" + + " ends with `.excluded` but keeps `folder/includeme.excluded`

" + + "`+` operator for globs is equivalent to `OR` operation.") + public Glob glob(Sequence include, Sequence exclude) throws EvalException { + List includeStrings = SkylarkUtil.convertStringList(include, "include"); + List excludeStrings = SkylarkUtil.convertStringList(exclude, "exclude"); + try { + return Glob.createGlob(includeStrings, excludeStrings); + } catch (IllegalArgumentException e) { + throw Starlark.errorf( + "Cannot create a glob from: include='%s' and exclude='%s': %s", + includeStrings, excludeStrings, e.getMessage()); + } + } + + @SuppressWarnings("unused") + @StarlarkMethod( + name = "parse_message", + doc = "Returns a ChangeMessage parsed from a well formed string.", + parameters = { + @Param( + name = "message", + named = true, + type = String.class, + doc = "The contents of the change message"), + }) + public ChangeMessage parseMessage(String changeMessage) throws EvalException { + try { + return ChangeMessage.parseMessage(changeMessage); + } catch (RuntimeException e) { + throw Starlark.errorf("Cannot parse change message '%s': %s", changeMessage, e.getMessage()); + } + } + + @StarlarkMethod( + name = "new_author", + doc = "Create a new author from a string with the form 'name '", + parameters = { + @Param( + name = "author_string", + type = String.class, + named = true, + doc = "A string representation of the author with the form 'name '"), + }) + @Example( + title = "Create a new author", + before = "", + code = "new_author('Foo Bar ')") + public Author newAuthor(String authorString) throws EvalException { + return Author.parse(authorString); + } +} diff --git a/third_party/copybara/java/com/google/copybara/Destination.java b/third_party/copybara/java/com/google/copybara/Destination.java new file mode 100644 index 0000000000..df9c85d33a --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Destination.java @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.util.Glob; +import com.google.copybara.util.console.Console; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.syntax.StarlarkValue; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Objects; +import javax.annotation.Nullable; + +/** A repository which a source of truth can be copied to. */ +@StarlarkBuiltin( + name = "destination", + doc = "A repository which a source of truth can be copied to", + category = StarlarkDocumentationCategory.TOP_LEVEL_TYPE, + documented = false) +public interface Destination extends ConfigItemDescription, StarlarkValue { + + /** + * An object which is capable of writing multiple revisions to the destination. This object is + * allowed to maintain state between the writing of revisions if applicable (for instance, to + * create multiple changes which are dependent on one another that require review before + * submission). + * + *

A single instance of this class is used to import either a single change, or a sequence of + * changes where each change is the following change's parent. + */ + interface Writer extends ChangeVisitable { + + /** + * Returns the status of the import at the destination. + * + *

This method may have undefined behavior if called after {@link #write(TransformResult, + * Glob, Console)}. + * + * @param labelName the label used in the destination for storing the last migrated ref + * @param destinationFiles the glob to use for filtering changes (optional) + */ + @Nullable + DestinationStatus getDestinationStatus(Glob destinationFiles, String labelName) + throws RepoException, ValidationException; + /** + * Returns true if this destination stores revisions in the repository so that + * {@link #getDestinationStatus(Glob, String)} can be used for discovering the state of the + * destination and we can use the methods in {@link ChangeVisitable}. + */ + boolean supportsHistory(); + + /** + * Writes the fully-transformed repository stored at {@code workdir} to this destination. + * @param transformResult what to write to the destination + * @param destinationFiles the glob to use for write. This glob might be different from the + * one received in {@code {@link #getDestinationStatus(Glob, String)}} due to read config from + * change configuration. + * @param console console to be used for printing messages + * @return one or more destination effects + * + * @throws ValidationException if an user attributable error happens during the write + * @throws RepoException if there was an issue with the destination repository + * @throws IOException if a file access error happens during the write + */ + ImmutableList write(TransformResult transformResult, Glob destinationFiles, + Console console) throws ValidationException, RepoException, IOException; + + /** + * Utility endpoint for accessing and adding feedback data. + * @param console console to use for reporting information to the user + */ + default Endpoint getFeedbackEndPoint(Console console) throws ValidationException { + return Endpoint.NOOP_ENDPOINT; + } + + default DestinationReader getDestinationReader( + Console console, @Nullable Origin.Baseline baseline, Path workdir) + throws ValidationException, RepoException { + return DestinationReader.NOT_IMPLEMENTED; + } + } + + /** + * Creates a writer which is capable of writing to this destination. This writer may maintain + * state between writing of revisions. + * + *

This method should only do trivial initialization of the writer, since it does not have + * access to a {@link Console}. + * + * @param writerContext Contains all the information for writing to destination, including + * workflowName, destinationFiles, * dryRun, revision, and oldWriter + * @throws ValidationException if the writer could not be created because of a user error. For + * instance, the destination cannot be used with the given {@code destinationFiles}. + */ + Writer newWriter(WriterContext writerContext) throws ValidationException; + + /** + * Given a reverse workflow with an {@code Origin} than is of the same type as this destination, + * the label that that {@link Origin#getLabelName()} would return. + * + *

This label name is used by the origin in the reverse workflow to stamp it's original + * revision id. Destinations return the origin label so that a baseline label can be found when + * using {@link WorkflowMode#CHANGE_REQUEST}. + */ + String getLabelNameWhenOrigin() throws ValidationException; + + /** + * This class represents the status of the destination. It includes the baseline revision + * and if it is a code review destination, the list of pending changes that have been already + * migrated. In order: First change is the oldest one. + */ + final class DestinationStatus { + + private final String baseline; + private final ImmutableList pendingChanges; + + public DestinationStatus(String baseline, ImmutableList pendingChanges) { + this.baseline = Preconditions.checkNotNull(baseline); + this.pendingChanges = Preconditions.checkNotNull(pendingChanges); + } + + /** + * String representation of the latest migrated revision in the baseline. + */ + public String getBaseline() { + return Preconditions.checkNotNull(baseline, "Trying to get baseline for NO_STATUS"); + } + + /** + * String representation of the migrated revisions that are in pending state in the destination. + * First element is the oldest one. Last element the newest one. + */ + public ImmutableList getPendingChanges() { + return Preconditions.checkNotNull(pendingChanges, + "Trying to get pendingChanges for NO_STATUS"); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DestinationStatus that = (DestinationStatus) o; + return Objects.equals(baseline, that.baseline) + && Objects.equals(pendingChanges, that.pendingChanges); + } + + @Override + public int hashCode() { + return Objects.hash(baseline, pendingChanges); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("baseline", baseline) + .add("pendingChanges", pendingChanges) + .toString(); + } + } + +} diff --git a/third_party/copybara/java/com/google/copybara/DestinationEffect.java b/third_party/copybara/java/com/google/copybara/DestinationEffect.java new file mode 100644 index 0000000000..f59bfe5707 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/DestinationEffect.java @@ -0,0 +1,343 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.Printer; +import com.google.devtools.build.lib.syntax.Sequence; +import com.google.devtools.build.lib.syntax.StarlarkList; +import com.google.devtools.build.lib.syntax.StarlarkValue; +import java.util.Objects; +import javax.annotation.Nullable; + +/** An effect happening in the destination as a consequence of the migration */ +@StarlarkBuiltin( + name = "destination_effect", + category = StarlarkDocumentationCategory.BUILTIN, + doc = "Represents an effect that happened in the destination due to a single migration") +@SuppressWarnings("unused") +public class DestinationEffect implements StarlarkValue { + private final Type type; + private final String summary; + private final ImmutableList originRefs; + @Nullable private final DestinationRef destinationRef; + private final ImmutableList errors; + + public DestinationEffect( + Type type, + String summary, + Iterable originRefs, + @Nullable DestinationRef destinationRef) { + this(type, summary, originRefs, destinationRef, ImmutableList.of()); + } + + public DestinationEffect( + Type type, + String summary, + Iterable originRefs, + @Nullable DestinationRef destinationRef, + Iterable errors) { + this.type = Preconditions.checkNotNull(type); + this.summary = Preconditions.checkNotNull(summary); + this.originRefs = ImmutableList.copyOf(Preconditions.checkNotNull(originRefs)); + this.destinationRef = destinationRef; + this.errors = ImmutableList.copyOf(Preconditions.checkNotNull(errors)); + } + + /** Returns the origin references included in this effect. */ + public ImmutableList getOriginRefs() { + return originRefs; + } + + @StarlarkMethod( + name = "origin_refs", + doc = "List of origin changes that were included in" + " this migration", + structField = true) + public final Sequence getOriginRefsSkylark() { + return StarlarkList.immutableCopyOf(originRefs); + } + + /** Return the type of effect that happened: Create, updated, noop or error */ + public Type getType() { + return type; + } + + @StarlarkMethod( + name = "type", + doc = + "Return the type of effect that happened: CREATED, UPDATED, NOOP, INSUFFICIENT_APPROVALS" + + " or ERROR", + structField = true + ) + public String getTypeSkylark() { + return type.toString(); + } + + /** Textual summary of what happened. Users of this class should not try to parse this field. */ + @StarlarkMethod( + name = "summary", + doc = + "Textual summary of what happened. Users of this class should not try to parse this" + + " field.", + structField = true + ) + public String getSummary() { + return summary; + } + + /** + * Destination reference updated/created. Might be null if there was no effect. Might be set even + * if the type is error (For example a synchronous presubmit test failed but a review was + * created). + */ + @StarlarkMethod( + name = "destination_ref", + doc = + "Destination reference updated/created. Might be null if there was no effect. Might be" + + " set even if the type is error (For example a synchronous presubmit test failed but" + + " a review was created).", + structField = true, + allowReturnNones = true + ) + @Nullable + public DestinationRef getDestinationRef() { + return destinationRef; + } + + /** + * List of errors that happened during the write to the destination. This can be used for example + * for synchronous presubmit failures. + */ + public ImmutableList getErrors() { + return errors; + } + + @StarlarkMethod( + name = "errors", + doc = "List of errors that happened during the migration", + structField = true) + public final Sequence getErrorsSkylark() { + return StarlarkList.immutableCopyOf(errors); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DestinationEffect that = (DestinationEffect) o; + return type == that.type + && Objects.equals(summary, that.summary) + && Objects.equals(originRefs, that.originRefs) + && Objects.equals(destinationRef, that.destinationRef) + && Objects.equals(errors, that.errors); + } + + @Override + public void repr(Printer printer) { + printer.append(toString()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("type", type) + .add("summary", summary) + .add("originRefs", originRefs) + .add("destinationRef", destinationRef) + .add("errors", errors) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(type, summary, originRefs, destinationRef, errors); + } + + /** Type of effect on the destination */ + public enum Type { + /** A new review or change was created */ + CREATED, + /** An existing review or change was updated */ + UPDATED, + /** + * The change was a noop. {@code destinationRef} might still be populated if the noop was + * detected against an existing review or pending change. + */ + NOOP, + /** The effect couldn't happen because the change doesn't have enough approvals */ + INSUFFICIENT_APPROVALS, + /** + * A user attributable error happened that prevented the destination from creating/updating the + * change. + */ + ERROR, + /** + * An error not attributable to the user that could be retried (RepoException, IOException...) + */ + TEMPORARY_ERROR, + /** + * A starting effect of a migration that is eventually expected to trigger another migration + * asynchronously. This allows to have 'dependant' migrations defined by users. + * An example of this: a workflow migrates code from a Gerrit review to a GitHub PR, and a + * feedback migration migrates the test results from a CI in GitHub back to the Gerrit change. + * This effect would be created on the former one. + */ + STARTED, + } + + /** Reference to the change/review read from the origin. */ + @StarlarkBuiltin( + name = "origin_ref", + category = StarlarkDocumentationCategory.BUILTIN, + doc = "Reference to the change/review in the origin.") + public static class OriginRef implements StarlarkValue { + private final String ref; + + @VisibleForTesting + public OriginRef(String id) { + this.ref = Preconditions.checkNotNull(id); + } + + /** Origin reference*/ + @StarlarkMethod(name = "ref", doc = "Origin reference ref", structField = true) + public String getRef() { + return ref; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OriginRef originRef = (OriginRef) o; + return Objects.equals(ref, originRef.ref); + } + + @Override + public int hashCode() { + return Objects.hashCode(ref); + } + + @Override + public void repr(Printer printer) { + printer.append(toString()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("ref", ref) + .toString(); + } + } + + /** Reference to the change/review created/updated on the destination. */ + @StarlarkBuiltin( + name = "destination_ref", + category = StarlarkDocumentationCategory.BUILTIN, + doc = "Reference to the change/review created/updated on the destination.") + public static class DestinationRef implements StarlarkValue { + @Nullable private final String url; + private final String id; + private final String type; + + public DestinationRef(String id, String type, @Nullable String url) { + this.id = Preconditions.checkNotNull(id); + this.type = Preconditions.checkNotNull(type); + this.url = url; + } + + /** Destination reference id */ + @StarlarkMethod(name = "id", doc = "Destination reference id", structField = true) + public String getId() { + return id; + } + + /** + * Type of reference created. Each destination defines its own and guarantees to be more stable + * than urls/ids. + */ + @StarlarkMethod( + name = "type", + doc = + "Type of reference created. Each destination defines its own and guarantees to be more" + + " stable than urls/ids", + structField = true + ) + public String getType() { + return type; + } + + /** Url, if any, of the destination change */ + @StarlarkMethod( + name = "url", + doc = "Url, if any, of the destination change", + structField = true, + allowReturnNones = true + ) + @Nullable + public String getUrl() { + return url; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DestinationRef that = (DestinationRef) o; + return Objects.equals(url, that.url) + && Objects.equals(id, that.id) + && Objects.equals(type, that.type); + } + + @Override + public int hashCode() { + return Objects.hash(url, id, type); + } + + @Override + public void repr(Printer printer) { + printer.append(toString()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("url", url) + .add("id", id) + .add("type", type) + .toString(); + } + } +} diff --git a/third_party/copybara/java/com/google/copybara/DestinationReader.java b/third_party/copybara/java/com/google/copybara/DestinationReader.java new file mode 100644 index 0000000000..70479247fd --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/DestinationReader.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.copybara.doc.annotations.Example; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.util.Glob; +import com.google.devtools.build.lib.skylarkinterface.Param; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.StarlarkValue; + +/** An api handle to read files from the destination, rather than just the origin. */ +@StarlarkBuiltin( + name = "destination_reader", + doc = "Handle to read from the destination", + category = StarlarkDocumentationCategory.TOP_LEVEL_TYPE, + documented = true) +public abstract class DestinationReader implements StarlarkValue { + + public static final DestinationReader NOT_IMPLEMENTED = new DestinationReader() { + @Override + public String readFile(String path) throws RepoException { + throw new RepoException("Reading files is not implemented by this destination"); + } + + @Override + public void copyDestinationFiles(Glob path) throws RepoException { + throw new RepoException("Reading files is not implemented by this destination"); + } + }; + + public static final DestinationReader NOOP_DESTINATION_READER = new DestinationReader() { + @Override + public String readFile(String path) { + return ""; + } + + @Override + public void copyDestinationFiles(Glob path) { + return; + } + }; + + @StarlarkMethod( + name = "read_file", + doc = "Read a file from the destination.", + parameters = { + @Param(name = "path", type = String.class, named = true, doc = "Path to the file."), + }) + @Example( + title = "Read a file from the destination's baseline", + before = "This can be added to the transformations of your core.workflow:", + code = + "def _read_destination_file(ctx):\n" + + " content = ctx.destination_reader().read_file(path = path/to/my_file.txt')\n" + + " ctx.console.info(content)\n\n" + + " transforms = [core.dynamic_transform(_read_destination_file)]\n", + after = + "Would print out the content of path/to/my_file.txt in the destination. The file does not" + + " have to be covered by origin_files nor destination_files.") + @SuppressWarnings("unused") + public abstract String readFile(String path) throws RepoException; + + @StarlarkMethod( + name = "copy_destination_files", + doc = "Copy files from the destination into the workdir.", + parameters = { + @Param(name = "glob", type = Glob.class, named = true, doc = "Files to copy to the " + + "workdir, potentially overwriting files checked out from the origin."), + }) + @Example( + title = "Copy files from the destination's baseline", + before = "This can be added to the transformations of your core.workflow:", + code = + "def _copy_destination_file(ctx):\n" + + " content = ctx.destination_reader().copy_destination_files(path = 'path/to/**')" + + "\n\n" + + " transforms = [core.dynamic_transform(_copy_destination_file)]\n", + after = + "Would copy all files in path/to/ from the destination baseline to the copybara workdir." + + " The files do not have to be covered by origin_files nor destination_files, but " + + "will cause errors if they are not covered by destination_files and not moved or " + + "deleted.") + @SuppressWarnings("unused") + public abstract void copyDestinationFiles(Glob glob) throws RepoException, ValidationException; +} diff --git a/third_party/copybara/java/com/google/copybara/DestinationStatusVisitor.java b/third_party/copybara/java/com/google/copybara/DestinationStatusVisitor.java new file mode 100644 index 0000000000..1dd336b831 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/DestinationStatusVisitor.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.copybara.ChangeVisitable.ChangesVisitor; +import com.google.copybara.ChangeVisitable.VisitResult; +import com.google.copybara.Destination.DestinationStatus; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import javax.annotation.Nullable; + +/** + * A visitor that computes the {@link DestinationStatus} matching the actual files affected by + * the changes with the destination files glob. + */ +public class DestinationStatusVisitor implements ChangesVisitor { + + private final PathMatcher pathMatcher; + private final String labelName; + + private DestinationStatus destinationStatus = null; + + public DestinationStatusVisitor(PathMatcher pathMatcher, String labelName) { + this.pathMatcher = pathMatcher; + this.labelName = labelName; + } + + @Override + public VisitResult visit(Change change) { + ImmutableSet changeFiles = change.getChangeFiles(); + if (changeFiles != null) { + if (change.getLabels().containsKey(labelName)) { + for (String file : changeFiles) { + if (pathMatcher.matches(Paths.get('/' + file))) { + String lastRev = Iterables.getLast(change.getLabels().get(labelName)); + destinationStatus = new DestinationStatus(lastRev, ImmutableList.of()); + return VisitResult.TERMINATE; + } + } + } + } + return VisitResult.CONTINUE; + } + + @Nullable + public DestinationStatus getDestinationStatus() { + return destinationStatus; + } +} diff --git a/third_party/copybara/java/com/google/copybara/Endpoint.java b/third_party/copybara/java/com/google/copybara/Endpoint.java new file mode 100644 index 0000000000..13857d6db1 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Endpoint.java @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.collect.ImmutableSetMultimap; +import com.google.copybara.DestinationEffect.DestinationRef; +import com.google.copybara.DestinationEffect.OriginRef; +import com.google.copybara.util.console.Console; +import com.google.devtools.build.lib.skylarkinterface.Param; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.EvalUtils; +import com.google.devtools.build.lib.syntax.Printer; +import com.google.devtools.build.lib.syntax.StarlarkValue; + +/** + * An origin or destination API in a feedback migration. + * + *

Endpoints are symmetric, that is, they need to be able to act both as an origin and + * destination of a feedback migration, which means that they need to support both read and write + * operations on the API. + */ +@SuppressWarnings("unused") +@StarlarkBuiltin( + name = "endpoint", + doc = "An origin or destination API in a feedback migration.", + category = StarlarkDocumentationCategory.TOP_LEVEL_TYPE) +public interface Endpoint extends StarlarkValue { + + /** + * To be used for core.workflow origin/destinations that don't want to provide an api for giving + * feedback. + */ + Endpoint NOOP_ENDPOINT = + new Endpoint() { + @Override + public ImmutableSetMultimap describe() { + throw new IllegalStateException("Instance shouldn't be used for core.feedback"); + } + + @Override + public void repr(Printer printer) { + printer.append("noop_endpoint"); + } + }; + + @Override + default void repr(Printer printer) { + printer.append(toString()); + } + + /** Returns a key-value ist of the options the endpoint was instantiated with. */ + ImmutableSetMultimap describe(); + + @StarlarkMethod( + name = "new_origin_ref", + doc = "Creates a new origin reference out of this endpoint.", + parameters = { + @Param(name = "ref", type = String.class, named = true, doc = "The reference."), + }) + default OriginRef newOriginRef(String ref) { + return new OriginRef(ref); + } + + @StarlarkMethod( + name = "new_destination_ref", + doc = "Creates a new destination reference out of this endpoint.", + parameters = { + @Param(name = "ref", type = String.class, named = true, doc = "The reference."), + @Param( + name = "type", + type = String.class, + named = true, + doc = "The type of this reference."), + @Param( + name = "url", + type = String.class, + named = true, + noneable = true, + doc = "The url associated with this reference, if any.", + defaultValue = "None"), + }) + default DestinationRef newDestinationRef(String ref, String type, Object urlObj) { + String url = EvalUtils.isNullOrNone(urlObj) ? null : (String) urlObj; + return new DestinationRef(ref, type, url); + } + + @StarlarkMethod( + name = "url", + doc = "Return the URL of this endpoint.", + structField = true, + allowReturnNones = true) + default String getUrl() { + return null; + } + + /** + * Returns an instance of this endpoint with the given console. + */ + default Endpoint withConsole(Console console) { + return this; + } +} diff --git a/third_party/copybara/java/com/google/copybara/EndpointProvider.java b/third_party/copybara/java/com/google/copybara/EndpointProvider.java new file mode 100644 index 0000000000..808b139e05 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/EndpointProvider.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.collect.ImmutableSetMultimap; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import com.google.devtools.build.lib.skylarkinterface.StarlarkDocumentationCategory; +import com.google.devtools.build.lib.skylarkinterface.StarlarkMethod; +import com.google.devtools.build.lib.syntax.StarlarkValue; + +/** Wrapper class to prevent arbitrary instantiation of endpoints in starlark. */ +@StarlarkBuiltin( + name = "endpoint_provider", + doc = "An handle for an origin or destination API in a feedback migration.", + category = StarlarkDocumentationCategory.TOP_LEVEL_TYPE) +public class EndpointProvider implements StarlarkValue, Endpoint { + final T endpoint; + + EndpointProvider(T endpoint) { + this.endpoint = endpoint; + } + + public T getEndpoint() { + return endpoint; + } + + @Override + public ImmutableSetMultimap describe() { + return endpoint.describe(); + } + + /** + * Wrap an Endpoint + */ + public static EndpointProvider wrap(T e) { + return new EndpointProvider<>(e); + } + + @Override + @StarlarkMethod( + name = "url", + doc = "Return the URL of this endpoint, if any.", + structField = true, + allowReturnNones = true) + public String getUrl() { + return endpoint.getUrl(); + } + +} diff --git a/third_party/copybara/java/com/google/copybara/FeedbackOptions.java b/third_party/copybara/java/com/google/copybara/FeedbackOptions.java new file mode 100644 index 0000000000..162a5d0729 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/FeedbackOptions.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.beust.jcommander.Parameters; + +/** Options for {@link com.google.copybara.feedback.Feedback} migrations. */ +@Parameters(separators = "=") +public class FeedbackOptions implements Option {} diff --git a/third_party/copybara/java/com/google/copybara/GeneralOptions.java b/third_party/copybara/java/com/google/copybara/GeneralOptions.java new file mode 100644 index 0000000000..9cd38b81af --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/GeneralOptions.java @@ -0,0 +1,430 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.copybara.exception.ValidationException.checkCondition; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ascii; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.base.Ticker; +import com.google.common.collect.ImmutableMap; +import com.google.common.flogger.FluentLogger; +import com.google.common.flogger.StackSize; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.jcommander.DurationConverter; +import com.google.copybara.jcommander.MapConverter; +import com.google.copybara.monitor.ConsoleEventMonitor; +import com.google.copybara.monitor.EventMonitor; +import com.google.copybara.profiler.Profiler; +import com.google.copybara.profiler.Profiler.ProfilerTask; +import com.google.copybara.util.CommandRunner; +import com.google.copybara.util.DirFactory; +import com.google.copybara.util.console.Console; +import com.google.copybara.util.console.StarlarkMode; +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.Callable; +import javax.annotation.Nullable; + +/** + * General options available for all the program classes. + */ +@Parameters(separators = "=") +public final class GeneralOptions implements Option { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + public static final String NOANSI = "--noansi"; + public static final String FORCE = "--force"; + public static final String CONFIG_ROOT_FLAG = "--config-root"; + public static final String OUTPUT_ROOT_FLAG = "--output-root"; + public static final String OUTPUT_LIMIT_FLAG = "--output-limit"; + public static final String DRY_RUN_FLAG = "--dry-run"; + public static final String SQUASH_FLAG = "--squash"; + static final Duration DEFAULT_CONSOLE_FILE_FLUSH_INTERVAL = Duration.ofSeconds(30); + + private Map environment; + private FileSystem fileSystem; + private Console console; + private EventMonitor eventMonitor; + private Path configRootPath; + private Path outputRootPath; + + private Profiler profiler = new Profiler(Ticker.systemTicker()); + + public GeneralOptions(Map environment, FileSystem fileSystem, Console console) { + this.environment = environment; + this.fileSystem = Preconditions.checkNotNull(fileSystem); + this.console = Preconditions.checkNotNull(console); + this.eventMonitor = new ConsoleEventMonitor(console, EventMonitor.EMPTY_MONITOR); + } + + @VisibleForTesting + public GeneralOptions(Map environment, FileSystem fileSystem, boolean verbose, + Console console, @Nullable Path configRoot, @Nullable Path outputRoot, + boolean noCleanup, boolean disableReversibleCheck, boolean force, int outputLimit) { + this.environment = ImmutableMap.copyOf(Preconditions.checkNotNull(environment)); + this.console = Preconditions.checkNotNull(console); + this.eventMonitor = new ConsoleEventMonitor(console, EventMonitor.EMPTY_MONITOR); + this.fileSystem = Preconditions.checkNotNull(fileSystem); + this.verbose = verbose; + this.configRootPath = configRoot; + this.outputRootPath = outputRoot; + this.noCleanup = noCleanup; + this.disableReversibleCheck = disableReversibleCheck; + this.force = force; + this.outputLimit = outputLimit; + } + + public GeneralOptions withForce(boolean force) throws ValidationException { + return new GeneralOptions(environment, fileSystem, verbose, console, getConfigRoot(), + getOutputRoot(), noCleanup, disableReversibleCheck, force, outputLimit); + } + + public GeneralOptions withConsole(Console console) throws ValidationException { + return new GeneralOptions(environment, fileSystem, verbose, console, getConfigRoot(), + getOutputRoot(), noCleanup, disableReversibleCheck, force, outputLimit); + } + + public Map getEnvironment() { + return environment; + } + + public boolean isVerbose() { + return verbose; + } + + public Console console() { + return console; + } + + public FileSystem getFileSystem() { + return fileSystem; + } + + public boolean isNoCleanup() { + return noCleanup; + } + + public boolean isDisableReversibleCheck() { + return disableReversibleCheck; + } + + public boolean isForced() { + return force; + } + + /** + * Returns current working directory + */ + public Path getCwd() { + return fileSystem.getPath(environment.get("PWD")); + } + + /** + * Returns the root absolute path to use for config. + */ + @Nullable + public Path getConfigRoot() throws ValidationException { + if (configRootPath == null && this.configRoot != null) { + configRootPath = fileSystem.getPath(this.configRoot).toAbsolutePath(); + checkCondition(Files.exists(configRootPath), "%s doesn't exist", configRoot); + checkCondition(Files.isDirectory(configRootPath), "%s isn't a directory", configRoot); + } + return configRootPath; + } + + /** + * Returns the output root directory, or null if not set. + * + *

This method is exposed mainly for tests and it's probably not what you're looking for. Try + * {@link #getDirFactory()} instead. + */ + @VisibleForTesting + @Nullable + public Path getOutputRoot() { + if (outputRootPath == null && this.outputRoot != null) { + outputRootPath = fileSystem.getPath(this.outputRoot); + } + return outputRootPath; + } + + /** + * Returns the output limit. + * + *

Each subcommand can use this value differently. + */ + public int getOutputLimit() { + return outputLimit > 0 ? outputLimit : Integer.MAX_VALUE; + } + + public Profiler profiler() { + return profiler; + } + + public EventMonitor eventMonitor() { + return eventMonitor; + } + + /** + * Run a repository task with profiling + */ + public T repoTask(String description, Callable callable) + throws RepoException, ValidationException { + try (ProfilerTask ignored = profiler().start(description)) { + return callable.call(); + } catch (Exception e) { + Throwables.propagateIfPossible(e, RepoException.class, ValidationException.class); + throw new RuntimeException("Unexpected exception", e); + } + } + + /** + * Run a repository task that can throw IOException with profiling + */ + public T ioRepoTask(String description, Callable callable) + throws RepoException, ValidationException, IOException{ + try (ProfilerTask ignored = profiler().start(description)) { + return callable.call(); + } catch (Exception e) { + Throwables.propagateIfPossible(e, RepoException.class, ValidationException.class); + Throwables.propagateIfPossible(e, IOException.class); + throw new RuntimeException("Unexpected exception", e); + } + } + + /** + * Returns a {@link DirFactory} capable of creating directories in a self contained location in + * the filesystem. + * + *

By default, the directories are created under {@code $HOME/copybara}, but it can be + * overridden with the flag --output-root. + */ + public DirFactory getDirFactory() { + if (getOutputRoot() != null) { + return new DirFactory(getOutputRoot()); + } else { + String home = checkNotNull(environment.get("HOME"), "$HOME environment var is not set"); + return new DirFactory(fileSystem.getPath(home).resolve("copybara")); + } + } + + @VisibleForTesting + public void setEnvironmentForTest(Map environment) { + this.environment = environment; + } + + @VisibleForTesting + public void setOutputRootPathForTest(Path outputRootPath) { + this.outputRootPath = outputRootPath; + } + + @VisibleForTesting + public void setConsoleForTest(Console console) { + this.console = console; + } + + @VisibleForTesting + public void setForceForTest(boolean force) { + this.force = force; + } + + @VisibleForTesting + public void setFileSystemForTest(FileSystem fileSystem) { + this.fileSystem = fileSystem; + } + + public GeneralOptions withProfiler(Profiler profiler) { + this.profiler = Preconditions.checkNotNull(profiler); + return this; + } + + public GeneralOptions withEventMonitor(EventMonitor eventMonitor) { + this.eventMonitor = new ConsoleEventMonitor(console(), eventMonitor); + return this; + } + + @Parameter( + names = {"-v", "--verbose"}, + description = "Verbose output.") + boolean verbose; + + @Parameter( + names = {"--fetch-timeout"}, + description = "Fetch timeout", + converter = DurationConverter.class) + public Duration fetchTimeout = CommandRunner.DEFAULT_TIMEOUT; + + // We don't use JCommander for parsing this flag but we do it manually since + // the parsing could fail and we need to report errors using one console + @SuppressWarnings("unused") + @Parameter(names = NOANSI, description = "Don't use ANSI output for messages") + boolean noansi = false; + + @Parameter( + names = FORCE, + description = + "Force the migration even if Copybara cannot find in the destination a change that is an" + + " ancestor of the one(s) being migrated. This should be used with care, as it" + + " could lose changes when migrating a previous/conflicting change.") + boolean force = false; + + @Parameter( + names = CONFIG_ROOT_FLAG, + description = + "Configuration root path to be used for resolving absolute config labels" + + " like '//foo/bar'") + String configRoot; + + @Parameter( + names = "--disable-reversible-check", + description = + "If set, all workflows will be executed without reversible_check, overriding" + + " the workflow config and the normal behavior for CHANGE_REQUEST mode.") + boolean disableReversibleCheck = false; + + @Parameter( + names = OUTPUT_ROOT_FLAG, + description = + "The root directory where to generate output files. If not set, ~/copybara/out is used " + + "by default. Use with care, Copybara might remove files inside this root if " + + "necessary.") + String outputRoot = null; + + @Parameter( + names = OUTPUT_LIMIT_FLAG, + description = + "Limit the output in the console to a number of records. Each subcommand might use this " + + "flag differently. Defaults to 0, which shows all the output.") + int outputLimit = 0; + + @Parameter( + names = "--nocleanup", + description = + "Cleanup the output directories. This includes the workdir, scratch clones of Git" + + " repos, etc. By default is set to false and directories will be cleaned prior to" + + " the execution. If set to true, the previous run output will not be cleaned up." + + " Keep in mind that running in this mode will lead to an ever increasing disk" + + " usage.") + boolean noCleanup = false; + + @Parameter( + names = "--nologging", + description = + "Disable logging of this binary. Note that commands executed by Copybara " + + "might still log to their own file.", hidden = true) + boolean noLogging = false; + + @Parameter( + names = "--temporary-features", + description = "Change guarded features. If set it means that it will return true.", + converter = MapConverter.class, + hidden = true) + private ImmutableMap temporaryFeatures = ImmutableMap.of(); + + + /** + * Temporary features is mean to be used by Copybara team for guarding new codepaths. Should + * never be used for user facing flags or longer term experiments. Any caller of this function + * should have a todo saying when to remove the call. + * + *

If the flag doesn't have a value it will use defaultVal. If the flag is incorrect (different + * from true/false) it will use defaultVal (And log at severe). + */ + public boolean isTemporaryFeature(String name, boolean defaultVal) { + Preconditions.checkNotNull(name); + String v = temporaryFeatures.get(name); + if (v == null) { + return defaultVal; + } + if (Ascii.equalsIgnoreCase(v, "true")) { + return true; + } + if (Ascii.equalsIgnoreCase(v, "false")) { + return false; + } + logger.atSevere() + .withStackTrace(StackSize.SMALL) + .log("Invalid boolean value for '%s' in '%s'. Needs to be true or false. Using default: %s", + name, temporaryFeatures, defaultVal); + return defaultVal; + } + + static final String CONSOLE_FILE_PATH = "--console-file-path"; + + // This flag is read before we parse the arguments, because of the console lifecycle + @SuppressWarnings("unused") + @Parameter( + names = CONSOLE_FILE_PATH, + description = "If set, write the console output also to the given file path.") + String consoleFilePath; + + // This flag is read before we parse the arguments, because of the console lifecycle + @SuppressWarnings("unused") + @Deprecated + @Parameter( + names = "--console-file-flush-rate", + description = + "How often in number of lines to flush the console to the output file. " + + "If set to 0, console will be flushed only at the end.", hidden = true) + int consoleFileFlushRateDeprecatedDontUse = -1; + + static final String CONSOLE_FILE_FLUSH_INTERVAL = "--console-file-flush-interval"; + + // This flag is read before we parse the arguments, because of the console lifecycle + @SuppressWarnings("unused") + @Parameter( + names = CONSOLE_FILE_FLUSH_INTERVAL, + converter = DurationConverter.class, + description = + "How often Copybara should flush the console to the output file. (10s, 1m, etc.)" + + "If set to 0s, console will be flushed only at the end.") + Duration consoleFileFlushInterval = DEFAULT_CONSOLE_FILE_FLUSH_INTERVAL; + + @Parameter(names = DRY_RUN_FLAG, + description = "Run the migration in dry-run mode. Some destination implementations might" + + " have some side effects (like creating a code review), but never submit to a main" + + " branch.") + public boolean dryRunMode = false; + + @Parameter(names = SQUASH_FLAG, description = "Override workflow's mode with 'SQUASH'. This is " + + "useful mainly for workflows that use 'ITERATIVE' mode, when we want to run a single " + + "export with 'SQUASH', maybe to fix an issue. Always use " + DRY_RUN_FLAG + " before, to " + + "test your changes locally.") + public boolean squash = false; + + @Parameter( + names = "--validate-starlark", + description = + "Starlark should be validated prior to execution, but this might break legacy configs." + + " Options are LOOSE, STRICT") + public String starlarkMode = StarlarkMode.LOOSE.name(); + + public StarlarkMode getStarlarkMode() { + return StarlarkMode.valueOf(starlarkMode); + } +} diff --git a/third_party/copybara/java/com/google/copybara/Info.java b/third_party/copybara/java/com/google/copybara/Info.java new file mode 100644 index 0000000000..8568fb9ef5 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Info.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; +import java.util.Optional; +import javax.annotation.Nullable; + +/** + * Represents the information about a Migration. + * + *

A migration can have one or more {@link MigrationReference}s. + */ +@AutoValue +public abstract class Info { + + public static final Info EMPTY = create(ImmutableMultimap.of(), + ImmutableMultimap.of(), ImmutableList.of()); + + public static Info create( + ImmutableMultimap originDescription, + ImmutableMultimap destinationDescription, + Iterable> migrationReferences) { + return new AutoValue_Info<>(originDescription, destinationDescription, + ImmutableList.copyOf(migrationReferences)); + } + + /** + * Returns origin description of the migration. + */ + public abstract ImmutableMultimap originDescription(); + + /** + * Returns destination description of the migration. + */ + public abstract ImmutableMultimap destinationDescription(); + + /** + * Returns information about a migration for one reference (like 'master') + * + *

Public so that it can be used programmatically. + */ + public abstract Iterable> migrationReferences(); + + @AutoValue + public abstract static class MigrationReference { + + public static MigrationReference create( + String label, + @Nullable O lastMigrated, + Iterable> availableToMigrate) { + return new AutoValue_Info_MigrationReference<>( + label, lastMigrated, ImmutableList.copyOf(availableToMigrate)); + } + + /** + * The name of this {@link MigrationReference}. + * + *

For a {@code Workflow} migration, the label is the string "workflow_" followed by the + * workflow name. + * + *

For a {@code Mirror} migration, the name is the string "mirror_" followed by the refspec. + */ + abstract String getLabel(); + + /** + * Returns the last migrated {@link Revision} from the origin, or {@code null} if no change was + * ever migrated. + */ + @Nullable + public abstract O getLastMigrated(); + + /** + * Returns the last available {@link Revision} to migrate from the origin, or {@code null} if + * there are no changes available to migrate. + * + *

There might be more available changes to migrate, but this is the revision of the most + * recent change available at this moment. + */ + @Nullable + public O getLastAvailableToMigrate() { + Optional lastAvailable = + getAvailableToMigrate() + .stream() + .map(Change::getRevision) + .reduce((first, second) -> second); + return lastAvailable.orElse(null); + } + + /** + * Returns a list of the next available {@link Change}s to migrate from the origin. + */ + public abstract ImmutableList> getAvailableToMigrate(); + + } +} diff --git a/third_party/copybara/java/com/google/copybara/InfoCmd.java b/third_party/copybara/java/com/google/copybara/InfoCmd.java new file mode 100644 index 0000000000..75b7f7c3d5 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/InfoCmd.java @@ -0,0 +1,166 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.beust.jcommander.Parameters; +import com.google.common.base.Ascii; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Iterables; +import com.google.copybara.Info.MigrationReference; +import com.google.copybara.config.Config; +import com.google.copybara.config.Migration; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.monitor.EventMonitor.InfoFinishedEvent; +import com.google.copybara.util.ExitCode; +import com.google.copybara.util.TablePrinter; +import com.google.copybara.util.console.Console; +import java.io.IOException; +import java.time.format.DateTimeFormatter; +import java.util.Comparator; + +/** + * Reads the last migrated revision in the origin and destination. + */ +@Parameters(separators = "=", + commandDescription = "Reads the last migrated revision in the origin and destination.") +public class InfoCmd implements CopybaraCmd { + + private static final int REVISION_MAX_LENGTH = 15; + private static final int DESCRIPTION_MAX_LENGTH = 80; + private static final int AUTHOR_MAX_LENGTH = 40; + private static final DateTimeFormatter DATE_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + private final ConfigLoaderProvider configLoaderProvider; + private final ContextProvider contextProvider; + + public InfoCmd(ConfigLoaderProvider configLoaderProvider, ContextProvider contextProvider) { + this.configLoaderProvider = Preconditions.checkNotNull(configLoaderProvider); + this.contextProvider = Preconditions.checkNotNull(contextProvider); + } + + @Override + public ExitCode run(CommandEnv commandEnv) + throws ValidationException, IOException, RepoException { + ConfigFileArgs configFileArgs = commandEnv.parseConfigFileArgs(this, /*useSourceRef*/false); + Console console = commandEnv.getOptions().get(GeneralOptions.class).console(); + Config config = configLoaderProvider + .newLoader(configFileArgs.getConfigPath(), configFileArgs.getSourceRef()) + .load(console); + if (configFileArgs.hasWorkflowName()) { + ImmutableMap context = + contextProvider.getContext(config, configFileArgs, configLoaderProvider, console); + info(commandEnv.getOptions(), config, configFileArgs.getWorkflowName(), context); + } else { + showAllMigrations(commandEnv, config); + } + return ExitCode.SUCCESS; + } + + private void showAllMigrations(CommandEnv commandEnv, Config config) { + TablePrinter table = new TablePrinter("Name", "Origin", "Destination", "Mode", "Description"); + for (Migration m : + config.getMigrations().values().stream() + .sorted(Comparator.comparing(Migration::getName)) + .collect(ImmutableList.toImmutableList())) { + table.addRow( + m.getName(), + prettyOriginDestination(m.getOriginDescription()), + prettyOriginDestination(m.getDestinationDescription()), + m.getModeString(), + Strings.nullToEmpty(m.getDescription())); + } + Console console = commandEnv.getOptions().get(GeneralOptions.class).console(); + for (String line : table.build()) { + console.info(line); + } + console.info("To get information about the state of any migration run:\n\n" + + " copybara info " + config.getLocation() + " [workflow_name]" + + "\n"); + } + + private static String prettyOriginDestination(ImmutableSetMultimap desc) { + return Iterables.getOnlyElement(desc.get("type")) + + (desc.containsKey("url") ? " (" + Iterables.getOnlyElement(desc.get("url")) + ")" : ""); + } + + /** Retrieves the {@link Info} of the {@code migrationName} and prints it to the console. */ + private static void info( + Options options, Config config, String migrationName, ImmutableMap context) + throws ValidationException, RepoException { + @SuppressWarnings("unchecked") + Info info = getInfo(migrationName, config); + Console console = options.get(GeneralOptions.class).console(); + int outputSize = 0; + for (MigrationReference migrationRef : info.migrationReferences()) { + console.info(String.format( + "'%s': last_migrated %s - last_available %s.", + migrationRef.getLabel(), + migrationRef.getLastMigrated() != null + ? migrationRef.getLastMigrated().asString() : "None", + migrationRef.getLastAvailableToMigrate() != null + ? migrationRef.getLastAvailableToMigrate().asString() : "None")); + + ImmutableList> availableToMigrate = + migrationRef.getAvailableToMigrate(); + int outputLimit = options.get(GeneralOptions.class).getOutputLimit(); + if (!availableToMigrate.isEmpty()) { + console.infoFmt( + "Available changes %s:", + availableToMigrate.size() <= outputLimit + ? String.format("(%d)", availableToMigrate.size()) + : String.format( + "(showing only first %d out of %d)", outputLimit, availableToMigrate.size())); + TablePrinter table = new TablePrinter("Date", "Revision", "Description", "Author"); + for (Change change : + Iterables.limit(availableToMigrate, outputLimit)) { + outputSize++; + table.addRow( + change.getDateTime().format(DATE_FORMATTER), + Ascii.truncate(change.getRevision().asString(), REVISION_MAX_LENGTH, ""), + Ascii.truncate(change.firstLineMessage(), DESCRIPTION_MAX_LENGTH, "..."), + Ascii.truncate(change.getAuthor().toString(), AUTHOR_MAX_LENGTH, "...")); + } + for (String line : table.build()) { + console.info(line); + } + } + if (outputSize > 100) { + console.infoFmt( + "Use %s to limit the output of the command.", GeneralOptions.OUTPUT_LIMIT_FLAG); + } + } + options.get(GeneralOptions.class).eventMonitor().onInfoFinished( + new InfoFinishedEvent(info, context)); + } + + /** Returns the {@link Info} of the {@code migrationName}. */ + private static Info getInfo(String migrationName, Config config) + throws ValidationException, RepoException { + return config.getMigration(migrationName).getInfo(); + } + + @Override + public String name() { + return "info"; + } +} diff --git a/third_party/copybara/java/com/google/copybara/LabelFinder.java b/third_party/copybara/java/com/google/copybara/LabelFinder.java new file mode 100644 index 0000000000..0fa5901687 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/LabelFinder.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.re2j.Matcher; +import com.google.re2j.Pattern; + +/** + * A simple line finder/parser for labels like: + *

    + *
  • foo = bar
    • + *
  • baz : foo
    • + *
+ * + *

In general this class should only be used in {@code Origin}s to create a labels map. + * During transformations/destination, it can be used to check if a line is a line but + * never to find labels. Use {@link TransformWork#getLabel(String)} instead, since it looks + * in more places for labels. + * + * TODO(malcon): Rename to MaybeLabel + */ +public class LabelFinder { + + private static final String VALID_LABEL_EXPR = "([\\w-]+)"; + + public static final Pattern VALID_LABEL = Pattern.compile(VALID_LABEL_EXPR); + + private static final Pattern URL = Pattern.compile(VALID_LABEL + "://.*"); + + private static final Pattern LABEL_PATTERN = Pattern.compile( + "^" + VALID_LABEL_EXPR + "( *[:=] ?)(.*)"); + private final Matcher matcher; + private final String line; + + public LabelFinder(String line) { + matcher = LABEL_PATTERN.matcher(line); + this.line = line; + } + + public boolean isLabel() { + // It is a line if it looks like a line but it doesn't look like a url (foo://bar) + return matcher.matches() && !URL.matcher(line).matches(); + } + + public boolean isLabel(String labelName) { + return isLabel() && getName().equals(labelName); + } + + /** + * Returns the name of the label. + * + *

Use isLabel() method before calling this method. + */ + public String getName() { + checkIsLabel(); + return matcher.group(1); + } + + /** + * Returns the separator of the label. + * + *

Use isLabel() method before calling this method. + */ + public String getSeparator() { + checkIsLabel(); + return matcher.group(2); + } + + /** + * Returns the value of the label. + * + *

Use isLabel() method before calling this method. + */ + public String getValue() { + checkIsLabel(); + return matcher.group(3); + } + + private void checkIsLabel() { + checkState(isLabel(), "Not a label: '" + line + "'. Please call isLabel() first"); + } + + public String getLine() { + return line; + } +} diff --git a/third_party/copybara/java/com/google/copybara/LazyResourceLoader.java b/third_party/copybara/java/com/google/copybara/LazyResourceLoader.java new file mode 100644 index 0000000000..414f138bdb --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/LazyResourceLoader.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.util.console.Console; +import javax.annotation.Nullable; + +/** + * Load a resource (repository, API client...) lazily to avoid side effects. + */ +public interface LazyResourceLoader { + + /** + * Load the resource. + */ + T load(@Nullable Console console) throws RepoException, ValidationException; + + /** + * Constructs a {@link LazyResourceLoader} object that defers the loading of the resource + * until {@link #load(Console)} is called and after that always returns the same instance. + */ + static LazyResourceLoader memoized(LazyResourceLoader delegate) { + + return new LazyResourceLoader() { + T resource; + + /** + * @see LazyResourceLoader#load(Console) + */ + @Override + public T load(Console console) throws RepoException, ValidationException { + if (resource == null) { + resource = Preconditions.checkNotNull(delegate.load(console)); + } + return resource; + } + }; + } +} diff --git a/third_party/copybara/java/com/google/copybara/LocalParallelizer.java b/third_party/copybara/java/com/google/copybara/LocalParallelizer.java new file mode 100644 index 0000000000..f259c9e325 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/LocalParallelizer.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.copybara.exception.ValidationException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; + +/** + * A class that allows to run a list of things in parallel batches. + */ +public class LocalParallelizer { + + private final int threads; + private final int minSize; + private final ListeningExecutorService executor; + + public LocalParallelizer(int threads, int minSize) { + this.threads = threads; + this.minSize = minSize; + Preconditions.checkState(threads >= 1, "Threads need to be positive"); + Preconditions.checkState(threads < 1000, "Too many threads (max: 1000)"); + executor = threads == 1 + ? null + : MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threads)); + } + + /** + * Run a list of things in batches, calling {@code func} for each batch. + */ + public List run(Iterable list, TransformFunc func) + throws IOException, ValidationException { + if (threads == 1 || Iterables.size(list) < minSize) { + return ImmutableList.of(func.run(list)); + } + List> results = new ArrayList<>(threads); + List newList = Lists.newArrayList(list); + for (List batch : Lists.partition(newList, Math.max(1, newList.size() / threads))) { + results.add(executor.submit(() -> func.run(batch))); + } + try { + return Futures.allAsList(results).get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + //TODO We cannot do much here. We might expose InterruptedException all the way up to Main... + throw new RuntimeException("Interrupted", e); + } catch (ExecutionException e) { + Throwables.propagateIfPossible(e.getCause(), IOException.class, ValidationException.class); + throw new RuntimeException("Unhandled error", e.getCause()); + + } + } + + /** Transforms a collection of K elements into T. */ + public interface TransformFunc { + + /** + * Execute oen batch. The number of elements is undefined. + */ + T run(Iterable elements) throws IOException, ValidationException; + } +} diff --git a/third_party/copybara/java/com/google/copybara/Main.java b/third_party/copybara/java/com/google/copybara/Main.java new file mode 100644 index 0000000000..06883a3b76 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Main.java @@ -0,0 +1,588 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import static com.google.copybara.MainArguments.COPYBARA_SKYLARK_CONFIG_FILENAME; +import static com.google.copybara.exception.ValidationException.checkCondition; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.StandardSystemProperty; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.flogger.FluentLogger; +import com.google.copybara.MainArguments.CommandWithArgs; +import com.google.copybara.config.ConfigValidator; +import com.google.copybara.config.Migration; +import com.google.copybara.config.PathBasedConfigFile; +import com.google.copybara.exception.CommandLineException; +import com.google.copybara.exception.EmptyChangeException; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.jcommander.DurationConverter; +import com.google.copybara.profiler.ConsoleProfilerListener; +import com.google.copybara.profiler.Listener; +import com.google.copybara.profiler.LogProfilerListener; +import com.google.copybara.profiler.Profiler; +import com.google.copybara.util.ExitCode; +import com.google.copybara.util.console.AnsiConsole; +import com.google.copybara.util.console.Console; +import com.google.copybara.util.console.FileConsole; +import com.google.copybara.util.console.LogConsole; +import com.google.devtools.build.lib.syntax.EvalException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.logging.LogManager; +import javax.annotation.Nullable; + +/** + * Main class that invokes Copybara from command-line. + * + *

This class should only know about how to validate and parse command-line arguments in order to + * invoke Copybara. + */ +public class Main { + + private static final String COPYBARA_NAMESPACE = "com.google.copybara"; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + /** + * Represents the environment, typically {@code System.getEnv()}. Injected to make easier tests. + * + *

Should not be mutated. + */ + protected final ImmutableMap environment; + protected Profiler profiler; + protected JCommander jCommander; + + private Console console; + + public Main() { + this(System.getenv()); + } + + public Main(Map environment) { + this.environment = Preconditions.checkNotNull(ImmutableMap.copyOf(environment)); + } + + public static void main(String[] args) { + System.exit(new Main(System.getenv()).run(args).getCode()); + } + + protected final ExitCode run(String[] args) { + // We need a console before parsing the args because it could fail with wrong + // arguments and we need to show the error. + this.console = getConsole(args); + // Configure logs location correctly before anything else. We want to write to the + // correct location in case of any error. + FileSystem fs = FileSystems.getDefault(); + try { + configureLog(fs, args); + } catch (IOException e) { + handleUnexpectedError(console, e.getMessage(), args, e); + return ExitCode.ENVIRONMENT_ERROR; + } + // This is useful when debugging user issues + logger.atInfo().log("Running: %s", Joiner.on(' ').join(args)); + + console.startupMessage(getVersion()); + + CommandResult result = runInternal(args, console, fs); + try { + shutdown(result); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + handleUnexpectedError(console, "Execution was interrupted.", args, e); + } + return result.exitCode; + } + + /** Helper to find out about verbose output before JCommander has been initialized .*/ + protected static boolean isVerbose(String[] args) { + return Arrays.stream(args).anyMatch(s -> s.equals("-v") || s.equals("--verbose")); + } + + /** Helper to find out if logging is enabled before JCommander has been initialized . */ + protected static boolean isEnableLogging(String[] args) { + return !Arrays.asList(args).contains("--nologging"); + } + + /** + * Finds a flag value before JCommander is initialized. Returns {@code Optional.empty()} if the + * flag is not present. + */ + protected static Optional findFlagValue(String[] args, String flagName) { + for (int index = 0; index < args.length - 1; index++) { + if (args[index].equals(flagName)) { + if (!args[index + 1].startsWith("-")) { + return Optional.of(args[index + 1]); + } + return Optional.empty(); + } + } + return Optional.empty(); + } + + /** A wrapper of the exit code and the command executed */ + protected static class CommandResult { + + private final ExitCode exitCode; + @Nullable private final CommandEnv commandEnv; + @Nullable private final CopybaraCmd command; + + CommandResult( + ExitCode exitCode, @Nullable CopybaraCmd command, @Nullable CommandEnv commandEnv) { + this.exitCode = Preconditions.checkNotNull(exitCode); + this.command = command; + this.commandEnv = commandEnv; + } + + public ExitCode getExitCode() { + return exitCode; + } + + /** + * The command environment passed to the command. Can be null for executions that failed before + * executing the command, like bad options. + */ + @Nullable + public CommandEnv getCommandEnv() { + return commandEnv; + } + + /** + * The command that was executed. Can be null for executions that failed before executing the + * command, like bad options. + */ + @Nullable + public CopybaraCmd getCommand() { + return command; + } + } + /** + * Runs the command and returns the {@link ExitCode}. + * + *

This method is also responsible for the exception handling/logging. + */ + private CommandResult runInternal(String[] args, Console console, FileSystem fs) { + CommandEnv commandEnv = null; + CopybaraCmd subcommand = null; + + try { + ModuleSet moduleSet = newModuleSet(environment, fs, console); + + final MainArguments mainArgs = new MainArguments(); + Options options = moduleSet.getOptions(); + jCommander = new JCommander(ImmutableList.builder() + .addAll(options.getAll()) + .add(mainArgs) + .build()); + jCommander.setProgramName("copybara"); + + String version = getVersion(); + logger.atInfo().log("Copybara version: %s", version); + jCommander.parse(args); + + + ConfigLoaderProvider configLoaderProvider = newConfigLoaderProvider(moduleSet); + + ImmutableMap commands = + Maps.uniqueIndex(getCommands(moduleSet, configLoaderProvider, jCommander), + CopybaraCmd::name); + // Tell jcommander about the commands; we don't actually use the feature, this is solely for + // generating the usage info. + for (Map.Entry cmd : commands.entrySet()) { + jCommander.addCommand(cmd.getKey(), cmd.getValue()); + } + CommandWithArgs cmdToRun = mainArgs.parseCommand(commands, commands.get("migrate")); + subcommand = cmdToRun.getSubcommand(); + + initEnvironment(options, cmdToRun.getSubcommand(), ImmutableList.copyOf(args)); + + GeneralOptions generalOptions = options.get(GeneralOptions.class); + Path baseWorkdir = mainArgs.getBaseWorkdir(generalOptions, generalOptions.getFileSystem()); + + commandEnv = new CommandEnv(baseWorkdir, options, cmdToRun.getArgs()); + generalOptions.console().progressFmt("Running %s", subcommand.name()); + + // TODO(malcon): Remove this after 2019-09-15, once tested that temp features work. + logger.atInfo().log("Temporary features test: %s", + options.get(GeneralOptions.class).isTemporaryFeature("TEST_TEMP_FEATURES", true)); + + ExitCode exitCode = subcommand.run(commandEnv); + return new CommandResult(exitCode, subcommand, commandEnv); + + } catch (CommandLineException | ParameterException e) { + printCauseChain(Level.WARNING, console, args, e); + console.error("Try 'copybara help'."); + return new CommandResult(ExitCode.COMMAND_LINE_ERROR, subcommand, commandEnv); + } catch (RepoException e) { + printCauseChain(Level.SEVERE, console, args, e); + // TODO(malcon): Expose interrupted exception from WorkflowMode to Main so that we don't + // have to do this hack. + if (e.getCause() instanceof InterruptedException) { + return new CommandResult(ExitCode.INTERRUPTED, subcommand, commandEnv); + } + return new CommandResult(ExitCode.REPOSITORY_ERROR, subcommand, commandEnv); + } catch (EmptyChangeException e) { + // This is not necessarily an error. Maybe the tool was run previously and there are no new + // changes to import. + console.warn(e.getMessage()); + return new CommandResult(ExitCode.NO_OP, subcommand, commandEnv); + } catch (ValidationException e) { + printCauseChain(Level.WARNING, console, args, e); + return new CommandResult(ExitCode.CONFIGURATION_ERROR, + subcommand, commandEnv); + } catch (IOException e) { + handleUnexpectedError(console, e.getMessage(), args, e); + return new CommandResult(ExitCode.ENVIRONMENT_ERROR, subcommand, commandEnv); + } catch (RuntimeException e) { + // This usually indicates a serious programming error that will require Copybara team + // intervention. Print stack trace without concern for presentation. + e.printStackTrace(); + handleUnexpectedError(console, + "Unexpected error (please file a bug against copybara): " + e.getMessage(), args, e); + return new CommandResult(ExitCode.INTERNAL_ERROR, subcommand, commandEnv); + } + } + + public ImmutableSet getCommands(ModuleSet moduleSet, + ConfigLoaderProvider configLoaderProvider, JCommander jcommander) + throws CommandLineException { + ConfigValidator validator = getConfigValidator(moduleSet.getOptions()); + Consumer consumer = getMigrationRanConsumer(); + return ImmutableSet.of( + new MigrateCmd(validator, consumer, configLoaderProvider), + new InfoCmd(configLoaderProvider, newInfoContextProvider()), + new ValidateCmd(validator, consumer, configLoaderProvider), + new HelpCmd(jcommander), + new VersionCmd()); + } + + /** + * Returns a short String representing the version of the binary + */ + protected String getVersion() { + return "Unknown version"; + } + + /** + * Returns a String (can be multiline) representing all the information about who and when the + * Copybara was built. + */ + protected String getBinaryInfo() { + return "Unknown version"; + } + + protected Consumer getMigrationRanConsumer() { + return migration -> {}; + } + + protected ConfigValidator getConfigValidator(Options options) throws CommandLineException { + return new ConfigValidator() {}; + } + + /** Returns a new module set. */ + protected ModuleSet newModuleSet(ImmutableMap environment, + FileSystem fs, Console console) { + return new ModuleSupplier(environment, fs, console).create(); + } + + protected ConfigLoaderProvider newConfigLoaderProvider(ModuleSet moduleSet) { + GeneralOptions generalOptions = moduleSet.getOptions().get(GeneralOptions.class); + return (configPath, sourceRef) -> new ConfigLoader(moduleSet, + createConfigFileWithHeuristic(validateLocalConfig(generalOptions, configPath), + generalOptions.getConfigRoot()), generalOptions.getStarlarkMode()); + } + + protected ContextProvider newInfoContextProvider() { + return (config, configFileArgs, configLoaderProvider, console) -> ImmutableMap.of(); + } + + /** + * Validate that the passed config file is correct (exists, follows the correct format, parent + * if passed is a real parent, etc.). + * + *

Returns the absolute {@link Path} of the config file. + */ + protected Path validateLocalConfig(GeneralOptions generalOptions, String configLocation) + throws ValidationException { + Path configPath = generalOptions.getFileSystem().getPath(configLocation).normalize(); + String fileName = configPath.getFileName().toString(); + checkCondition( + fileName.contentEquals(COPYBARA_SKYLARK_CONFIG_FILENAME), + "Copybara config file filename should be '%s' but it is '%s'.", + COPYBARA_SKYLARK_CONFIG_FILENAME, configPath.getFileName()); + + // Treat the top level element specially since it is passed thru the command line. + if (!Files.exists(configPath)) { + throw new CommandLineException("Configuration file not found: " + configPath); + } + return configPath.toAbsolutePath(); + } + + /** + * Find the root path for resolving configuration file paths and resources. This method assumes + * that the .git containing directory is the root path. + * + *

This could be extended to other kind of source control systems. + */ + protected PathBasedConfigFile createConfigFileWithHeuristic( + Path configPath, @Nullable Path commandLineRoot) { + if (commandLineRoot != null) { + return new PathBasedConfigFile(configPath, commandLineRoot, /*identifierPrefix=*/ null); + } + Path parent = configPath.getParent(); + while (parent != null) { + if (Files.isDirectory(parent.resolve(".git"))) { + return new PathBasedConfigFile(configPath, parent, /*identifierPrefix=*/ null); + } + parent = parent.getParent(); + } + return new PathBasedConfigFile(configPath, /*rootPath=*/ null, /*identifierPrefix=*/ null); + } + + protected Console getConsole(String[] args) { + boolean verbose = isVerbose(args); + // If System.console() is not present, we are forced to use LogConsole + Console console; + if (System.console() == null) { + console = LogConsole.writeOnlyConsole(System.err, verbose); + } else if (Arrays.asList(args).contains(GeneralOptions.NOANSI)) { + // The System.console doesn't detect redirects/pipes, but at least we have + // jobs covered. + console = LogConsole.readWriteConsole(System.in, System.err, verbose); + } else { + console = new AnsiConsole(System.in, System.err, verbose); + } + Optional maybeConsoleFilePath = findFlagValue(args, GeneralOptions.CONSOLE_FILE_PATH); + if (!maybeConsoleFilePath.isPresent()) { + return console; + } + Path consoleFilePath = Paths.get(maybeConsoleFilePath.get()); + try { + Files.createDirectories(consoleFilePath.getParent()); + } catch (IOException e) { + logger.atSevere().withCause(e).log( + "Could not create parent directories to file: %s. Redirecting will be disabled.", + consoleFilePath); + return console; + } + return new FileConsole(console, consoleFilePath, getConsoleFlushRate(args)); + } + + /** + * Returns the console flush rate from the flag, if present and valid, or 0 (no flush) otherwise. + */ + protected Duration getConsoleFlushRate(String[] args) { + return findFlagValue(args, GeneralOptions.CONSOLE_FILE_FLUSH_INTERVAL) + .map(e -> new DurationConverter().convert(e)) + .orElse(GeneralOptions.DEFAULT_CONSOLE_FILE_FLUSH_INTERVAL); + } + + protected void configureLog(FileSystem fs, String[] args) throws IOException { + String baseDir = getBaseExecDir(); + Files.createDirectories(fs.getPath(baseDir)); + if (System.getProperty("java.util.logging.config.file") == null) { + logger.atInfo().log("Setting up LogManager"); + String level = isEnableLogging(args) ? "INFO" : "OFF"; + LogManager.getLogManager().readConfiguration(new ByteArrayInputStream(( + "handlers=java.util.logging.FileHandler\n" + + ".level=INFO\n" + + "java.util.logging.FileHandler.level=" + level +"\n" + + "java.util.logging.FileHandler.pattern=" + + baseDir + "/copybara-%g.log\n" + + "java.util.logging.FileHandler.count=10\n" + + "java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter\n" + + "java.util.logging.SimpleFormatter.format=" + + "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %4$-6s %2$s %5$s%6$s%n") + .getBytes(StandardCharsets.UTF_8) + )); + } + } + + /** + * Hook to allow setting variables that are not run or validation specific, based on options. + * Sample use case are remote logging, test harnesses and others. Called after command line + * options are parsed, but before a file is read or a run started. + */ + protected void initEnvironment(Options options, CopybaraCmd copybaraCmd, + ImmutableList rawArgs) + throws ValidationException, IOException, RepoException { + GeneralOptions generalOptions = options.get(GeneralOptions.class); + profiler = generalOptions.profiler(); + ImmutableList.Builder profilerListeners = ImmutableList.builder(); + profilerListeners.add( + new LogProfilerListener(), new ConsoleProfilerListener(generalOptions.console())); + profiler.init(profilerListeners.build()); + cleanupOutputDir(generalOptions); + } + + protected void cleanupOutputDir(GeneralOptions generalOptions) + throws RepoException, IOException, ValidationException { + generalOptions + .ioRepoTask( + "clean_outputdir", + () -> { + if (generalOptions.isNoCleanup()) { + return null; + } + generalOptions.console().progress("Cleaning output directory"); + generalOptions.getDirFactory().cleanupTempDirs(); + // Only for profiling purposes, no need to use the console + logger.atInfo() + .log("Cleaned output directory:" + generalOptions.getDirFactory().getTmpRoot()); + return null; + }); + } + /** + * Performs cleanup tasks after executing Copybara. + * @param result + */ + protected void shutdown(CommandResult result) throws InterruptedException { + // Before profiler.stop() + if (console != null) { + console.close(); + } + if (profiler != null) { + profiler.stop(); + } + } + + /** + * Returns the base directory to be used by Copybara to write execution related files (Like + * logs). + */ + private String getBaseExecDir() { + // In this case we are not using GeneralOptions.getEnvironment() because we still haven't built + // the options, but it's fine. This is the tool's Main and is also injecting System.getEnv() + // to the options, so the value is the same. + String userHome = StandardSystemProperty.USER_HOME.value(); + + switch (StandardSystemProperty.OS_NAME.value()) { + case "Linux": + String xdgCacheHome = System.getenv("XDG_CACHE_HOME"); + return Strings.isNullOrEmpty(xdgCacheHome) + ? userHome + "/.cache/" + COPYBARA_NAMESPACE + : xdgCacheHome + COPYBARA_NAMESPACE; + case "Mac OS X": + return userHome + "/Library/Logs/" + COPYBARA_NAMESPACE; + default: + return "/var/tmp/" + COPYBARA_NAMESPACE; + } + } + + private void printCauseChain(Level level, Console console, String[] args, Throwable e) { + StringBuilder error = new StringBuilder(e.getMessage()).append("\n"); + Throwable cause = e.getCause(); + while (cause != null) { + error.append(" CAUSED BY: ").append(printException(cause)).append("\n"); + cause = cause.getCause(); + } + console.error(error.toString()); + logger.at(level).withCause(e).log(formatLogError(e.getMessage(), args)); + } + + private String printException(Throwable t) { + if (t instanceof EvalException) { + return (((EvalException) t).print()); + } + return t.getMessage(); + } + + private void handleUnexpectedError(Console console, String msg, String[] args, Throwable e) { + logger.atSevere().withCause(e).log(formatLogError(msg, args)); + console.error(msg + " (" + e + ")"); + } + + private static String usage(JCommander jcommander, String version) { + StringBuilder fullUsage = new StringBuilder(); + fullUsage.append("Copybara version: ").append(version).append("\n"); + jcommander.usage(fullUsage); + fullUsage + .append("\n") + .append("Example:\n") + .append(" copybara ").append(COPYBARA_SKYLARK_CONFIG_FILENAME).append(" origin/master\n"); + return fullUsage.toString(); + } + + private static String formatLogError(String message, String[] args) { + return String.format("%s (command args: %s)", message, Arrays.toString(args)); + } + + /** Prints the Copybara version */ + @Parameters(separators = "=", commandDescription = "Shows the version of Copybara.") + private class VersionCmd implements CopybaraCmd { + + @Override + public ExitCode run(CommandEnv commandEnv) + throws ValidationException, IOException, RepoException { + commandEnv.getOptions().get(GeneralOptions.class).console().info(getBinaryInfo()); + return ExitCode.SUCCESS; + } + + @Override + public String name() { + return "version"; + } + } + + /** + * Prints the help message + * TODO(malcon): Implement help per command + */ + @Parameters(separators = "=", commandDescription = "Shows the help.") + private class HelpCmd implements CopybaraCmd { + + private final JCommander jCommander; + + HelpCmd(JCommander jCommander) { + this.jCommander = Preconditions.checkNotNull(jCommander); + } + + @Override + public ExitCode run(CommandEnv commandEnv) + throws ValidationException, IOException, RepoException { + String version = getVersion(); + commandEnv.getOptions().get(GeneralOptions.class).console().info(usage(jCommander, version)); + return ExitCode.SUCCESS; + } + + @Override + public String name() { + return "help"; + } + } +} diff --git a/third_party/copybara/java/com/google/copybara/MainArguments.java b/third_party/copybara/java/com/google/copybara/MainArguments.java new file mode 100644 index 0000000000..fb15d2dd80 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/MainArguments.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.copybara.exception.CommandLineException; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.concurrent.NotThreadSafe; + +/** + * Arguments which are unnamed (i.e. positional) or must be evaluated inside {@link Main}. + */ +@NotThreadSafe +@Parameters(separators = "=") +public final class MainArguments { + private final Logger logger = Logger.getLogger(this.getClass().getName()); + + public static final String COPYBARA_SKYLARK_CONFIG_FILENAME = "copy.bara.sky"; + + // TODO(danielromero): Annotate the subcommands with the documentation and generate this + // automatically. + @Parameter(description = + "" + + "[subcommand] [config_path [migration_name [source_ref]...]]\n" + + "\n" + + ("" + + "subcommand: Optional, defaults to 'migrate'. The type of task to be performed by " + + "Copybara. Available subcommands:\n" + + " - help: Shows the help.\n" + + " - info: Reads the last migrated revision in the origin and destination.\n" + + " - migrate: Executes the migration for the given config.\n" + + " - validate: Validates that the configuration is correct.\n" + + " - version: Shows the version of Copybara.\n" + + "") + + "\n" + + "config_path: Required. Relative or absolute path to the main Copybara config file.\n" + + "\n" + + "migration_name: Optional, defaults to 'default'. The name of the migration that the " + + "subcommand will be applied to.\n" + + "\n" + + "source_ref: Optional. The reference(s) to be resolved in the origin. Most of the " + + "times this argument is not needed, as Copybara can infer the last migrated reference " + + "in the destination. Different subcommands might require this argument, use only one " + + "source_ref or use all the list.\n" + ) + List unnamed = new ArrayList<>(); + + @Parameter(names = "--work-dir", description = "Directory where all the transformations" + + " will be performed. By default a temporary directory.") + String baseWorkdir; + + /** + * Returns the base working directory. This method should not be accessed directly by any other + * class but Main. + */ + public Path getBaseWorkdir(GeneralOptions generalOptions, FileSystem fs) + throws IOException { + Path workdirPath; + + workdirPath = baseWorkdir == null + ? generalOptions.getDirFactory().newTempDir("workdir") + : fs.getPath(baseWorkdir).normalize(); + logger.log(Level.INFO, String.format("Using workdir: %s", workdirPath.toAbsolutePath())); + + if (Files.exists(workdirPath) && !Files.isDirectory(workdirPath)) { + // Better being safe + throw new IOException( + "'" + workdirPath + "' exists and is not a directory"); + } + if (!isDirEmpty(workdirPath)) { + System.err.println("WARNING: " + workdirPath + " is not empty"); + } + return workdirPath; + } + + private static boolean isDirEmpty(final Path directory) throws IOException { + try (DirectoryStream dirStream = Files.newDirectoryStream(directory)) { + return !dirStream.iterator().hasNext(); + } + } + + CommandWithArgs parseCommand(ImmutableMap commands, + CopybaraCmd defaultCmd) throws CommandLineException { + if (unnamed.isEmpty()) { + return new CommandWithArgs(defaultCmd, ImmutableList.of()); + } + String firstArg = unnamed.get(0); + // Default command might take a config file as param. + if (firstArg.endsWith(COPYBARA_SKYLARK_CONFIG_FILENAME)) { + return new CommandWithArgs(defaultCmd, ImmutableList.copyOf(unnamed)); + } + + if (!commands.containsKey(firstArg.toLowerCase())) { + throw new CommandLineException( + String.format("Invalid subcommand '%s'. Available commands: %s", firstArg, + new TreeSet<>(commands.keySet()))); + } + return new CommandWithArgs(commands.get(firstArg.toLowerCase()), + ImmutableList.copyOf(unnamed.subList(1, unnamed.size()))); + } + + static class CommandWithArgs { + + private final CopybaraCmd subcommand; + private final ImmutableList args; + + private CommandWithArgs(CopybaraCmd subcommand, ImmutableList args) { + this.subcommand = Preconditions.checkNotNull(subcommand); + this.args = Preconditions.checkNotNull(args); + } + + CopybaraCmd getSubcommand() { + return subcommand; + } + + ImmutableList getArgs() { + return args; + } + } +} diff --git a/third_party/copybara/java/com/google/copybara/MapMapper.java b/third_party/copybara/java/com/google/copybara/MapMapper.java new file mode 100644 index 0000000000..a58b7c993b --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/MapMapper.java @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableMap; +import com.google.copybara.transform.ReversibleFunction; +import com.google.devtools.build.lib.syntax.Location; + +public class MapMapper implements ReversibleFunction { + + private final ImmutableMap map; + private final Location location; + + MapMapper(ImmutableMap map, Location location) { + this.map = Preconditions.checkNotNull(map); + this.location = location; + } + + @Override + public ReversibleFunction reverseMapping() throws NonReversibleValidationException { + try { + return new MapMapper(ImmutableBiMap.copyOf(map).inverse(), location); + } catch (IllegalArgumentException e) { + throw new NonReversibleValidationException(location, + "Non-reversible map: " + map + ": " + e.getMessage()); + } + } + + @Override + public String apply(String s) { + String v = map.get(s); + return v == null ? s : v; + } +} diff --git a/third_party/copybara/java/com/google/copybara/Metadata.java b/third_party/copybara/java/com/google/copybara/Metadata.java new file mode 100644 index 0000000000..a997143cee --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Metadata.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.copybara.authoring.Author; + +/** + * Metadata associated with a change: Change message, author, etc. + */ +public final class Metadata { + + private final String message; + private final Author author; + private final ImmutableSetMultimap hiddenLabels; + + public Metadata(String message, Author author, + ImmutableSetMultimap hiddenLabels) { + this.message = checkNotNull(message); + this.author = checkNotNull(author); + this.hiddenLabels = checkNotNull(hiddenLabels); + } + + public final Metadata withAuthor(Author author) { + return new Metadata(message, checkNotNull(author, "Author cannot be null"), hiddenLabels); + } + + public final Metadata withMessage(String message) { + return new Metadata(checkNotNull(message, "Message cannot be null"), author, hiddenLabels); + } + + /** + * We never allow deleting hidden labels. Use a different name if you want to rename one. + */ + public final Metadata addHiddenLabels(ImmutableMultimap hiddenLabels) { + checkNotNull(hiddenLabels, "hidden labels cannot be null"); + return new Metadata(message, author, + ImmutableSetMultimap.builder() + .putAll(this.hiddenLabels) + .putAll(hiddenLabels).build()); + } + + /** + * Description to be used for the change + */ + public String getMessage() { + return message; + } + + /** + * Author to be used for the change + */ + public Author getAuthor() { + return author; + } + + /** + * Hidden labels are labels added by transformations during transformations but that they are + * not visible in the message. + */ + public ImmutableSetMultimap getHiddenLabels() { + return hiddenLabels; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("message", message) + .add("author", author) + .add("hiddenLabels", hiddenLabels) + .toString(); + } +} diff --git a/third_party/copybara/java/com/google/copybara/MigrateCmd.java b/third_party/copybara/java/com/google/copybara/MigrateCmd.java new file mode 100644 index 0000000000..a9b1cc5133 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/MigrateCmd.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import static com.google.copybara.exception.ValidationException.checkCondition; + +import com.beust.jcommander.Parameters; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.copybara.config.Config; +import com.google.copybara.config.ConfigValidator; +import com.google.copybara.config.Migration; +import com.google.copybara.config.ValidationResult; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.util.ExitCode; +import com.google.copybara.util.console.Console; +import java.io.IOException; +import java.nio.file.Path; +import java.util.function.Consumer; + +/** + * Executes the migration for the given config. + */ +@Parameters(separators = "=", commandDescription = "Executes the migration for the given config.") +public class MigrateCmd implements CopybaraCmd { + + private final ConfigValidator configValidator; + private final Consumer migrationRanConsumer; + private final ConfigLoaderProvider configLoaderProvider; + + MigrateCmd(ConfigValidator configValidator, Consumer migrationRanConsumer, + ConfigLoaderProvider configLoaderProvider) { + this.configValidator = Preconditions.checkNotNull(configValidator); + this.migrationRanConsumer = Preconditions.checkNotNull(migrationRanConsumer); + this.configLoaderProvider = Preconditions.checkNotNull(configLoaderProvider); + } + + @Override + public ExitCode run(CommandEnv commandEnv) + throws RepoException, ValidationException, IOException { + ConfigFileArgs configFileArgs = commandEnv.parseConfigFileArgs(this, + /*useSourceRef*/true); + ImmutableList sourceRefs = configFileArgs.getSourceRefs(); + run( + commandEnv.getOptions(), + configLoaderProvider.newLoader( + configFileArgs.getConfigPath(), + sourceRefs.size() == 1 ? Iterables.getOnlyElement(sourceRefs) : null), + configFileArgs.getWorkflowName(), + commandEnv.getWorkdir(), + sourceRefs); + return ExitCode.SUCCESS; + } + + /** + * Runs the migration specified by {@code migrationName}. + */ + private void run(Options options, ConfigLoader configLoader, String migrationName, + Path workdir, ImmutableList sourceRefs) + throws RepoException, ValidationException, IOException { + Config config = loadConfig(options, configLoader, migrationName); + Migration migration = config.getMigration(migrationName); + + if (!options.get(WorkflowOptions.class).isReadConfigFromChange()) { + this.migrationRanConsumer.accept(migration); + migration.run(workdir, sourceRefs); + return; + } + + checkCondition(configLoader.supportsLoadForRevision(), + "%s flag is not supported for the origin/config file path", + WorkflowOptions.READ_CONFIG_FROM_CHANGE); + + // A safeguard, mirror workflows are not supported in the service anyway + checkCondition(migration instanceof Workflow, + "Flag --read-config-from-change is not supported for non-workflow migrations: %s", + migrationName); + migrationRanConsumer.accept(migration); + @SuppressWarnings("unchecked") + Workflow workflow = + (Workflow) migration; + new ReadConfigFromChangeWorkflow<>(workflow, options, configLoader, configValidator) + .run(workdir, sourceRefs); + } + + private Config loadConfig(Options options, ConfigLoader configLoader, String migrationName) + throws IOException, ValidationException { + GeneralOptions generalOptions = options.get(GeneralOptions.class); + Console console = generalOptions.console(); + Config config = configLoader.load(console); + console.progress("Validating configuration"); + ValidationResult result = configValidator.validate(config, migrationName); + if (!result.hasErrors()) { + return config; + } + result.getErrors().forEach(console::error); + console.error("Configuration is invalid."); + throw new ValidationException("Error validating configuration: Configuration is invalid."); + } + + @Override + public String name() { + return "migrate"; + } +} diff --git a/third_party/copybara/java/com/google/copybara/MigrationInfo.java b/third_party/copybara/java/com/google/copybara/MigrationInfo.java new file mode 100644 index 0000000000..9171c16ee3 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/MigrationInfo.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + + +import static com.google.common.base.Preconditions.checkNotNull; + +import javax.annotation.Nullable; + +/** + * Reflective information about the migration in progress. + */ +public class MigrationInfo { + @Nullable private final String originLabel; + @Nullable private final ChangeVisitable destinationVisitable; + + public MigrationInfo(String originLabel, ChangeVisitable destinationVisitable) { + this.originLabel = originLabel; + this.destinationVisitable = destinationVisitable; + } + + public String getOriginLabel() { + return checkNotNull(originLabel); + } + + @Nullable + public ChangeVisitable destinationVisitable() { + return destinationVisitable; + } +} diff --git a/third_party/copybara/java/com/google/copybara/ModuleSet.java b/third_party/copybara/java/com/google/copybara/ModuleSet.java new file mode 100644 index 0000000000..6efdfd9c84 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/ModuleSet.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * A set of modules and options for evaluating a Skylark config file. + */ +public class ModuleSet { + + private final Options options; + // TODO(malcon): Remove this once all modules are @StarlarkMethod + private final ImmutableSet> staticModules; + private final ImmutableMap modules; + + ModuleSet(Options options, + ImmutableSet> staticModules, + ImmutableMap modules) { + this.options = Preconditions.checkNotNull(options); + this.staticModules = Preconditions.checkNotNull(staticModules); + this.modules = Preconditions.checkNotNull(modules); + } + + /** + * Copybara options + */ + public Options getOptions() { + return options; + } + + /** + * Static modules. Will be deleted. + * TODO(malcon): Delete + */ + public ImmutableSet> getStaticModules() { + return staticModules; + } + + /** + * Non-static Copybara modules. + */ + public ImmutableMap getModules() { + return modules; + } + + +} diff --git a/third_party/copybara/java/com/google/copybara/ModuleSupplier.java b/third_party/copybara/java/com/google/copybara/ModuleSupplier.java new file mode 100644 index 0000000000..b50f5e4d86 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/ModuleSupplier.java @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.copybara.authoring.Authoring; +import com.google.copybara.buildozer.BuildozerModule; +import com.google.copybara.buildozer.BuildozerOptions; +import com.google.copybara.folder.FolderDestinationOptions; +import com.google.copybara.folder.FolderModule; +import com.google.copybara.folder.FolderOriginOptions; +import com.google.copybara.format.BuildifierOptions; +import com.google.copybara.format.FormatModule; +import com.google.copybara.git.GerritOptions; +import com.google.copybara.git.GitDestinationOptions; +import com.google.copybara.git.GitHubDestinationOptions; +import com.google.copybara.git.GitHubOptions; +import com.google.copybara.git.GitHubPrOriginOptions; +import com.google.copybara.git.GitMirrorOptions; +import com.google.copybara.git.GitModule; +import com.google.copybara.git.GitOptions; +import com.google.copybara.git.GitOriginOptions; +import com.google.copybara.hg.HgModule; +import com.google.copybara.hg.HgOptions; +import com.google.copybara.hg.HgOriginOptions; +import com.google.copybara.remotefile.RemoteFileModule; +import com.google.copybara.remotefile.RemoteFileOptions; +import com.google.copybara.transform.debug.DebugOptions; +import com.google.copybara.transform.metadata.MetadataModule; +import com.google.copybara.transform.patch.PatchModule; +import com.google.copybara.transform.patch.PatchingOptions; +import com.google.copybara.util.console.Console; +import com.google.devtools.build.lib.skylarkinterface.StarlarkBuiltin; +import java.nio.file.FileSystem; +import java.util.Map; +import java.util.function.Function; + +/** + * A supplier of modules and {@link Option}s for Copybara. + */ +public class ModuleSupplier { + + private static final ImmutableSet> BASIC_MODULES = ImmutableSet.of( + CoreGlobal.class); + private final Map environment; + private final FileSystem fileSystem; + private final Console console; + + public ModuleSupplier(Map environment, FileSystem fileSystem, + Console console) { + this.environment = Preconditions.checkNotNull(environment); + this.fileSystem = Preconditions.checkNotNull(fileSystem); + this.console = Preconditions.checkNotNull(console); + } + + /** + * Returns the {@code set} of modules available. + * TODO(malcon): Remove once no more static modules exist. + */ + protected ImmutableSet> getStaticModules() { + return BASIC_MODULES; + } + + /** + * Get non-static modules available + */ + public ImmutableSet getModules(Options options) { + GeneralOptions general = options.get(GeneralOptions.class); + return ImmutableSet.of( + new Core(general, options.get(WorkflowOptions.class), options.get(DebugOptions.class)), + new GitModule(options), new HgModule(options), + new FolderModule( + options.get(FolderOriginOptions.class), + options.get(FolderDestinationOptions.class), + general), + new FormatModule( + options.get(WorkflowOptions.class), options.get(BuildifierOptions.class), general), + new BuildozerModule( + options.get(WorkflowOptions.class), options.get(BuildozerOptions.class)), + new PatchModule(options.get(PatchingOptions.class)), + new MetadataModule(), + new Authoring.Module(), + new RemoteFileModule(options)); + } + + /** Returns a new list of {@link Option}s. */ + protected Options newOptions() { + GeneralOptions generalOptions = new GeneralOptions(environment, fileSystem, console); + GitOptions gitOptions = new GitOptions(generalOptions); + GitDestinationOptions gitDestinationOptions = + new GitDestinationOptions(generalOptions, gitOptions); + BuildifierOptions buildifierOptions = new BuildifierOptions(); + WorkflowOptions workflowOptions = new WorkflowOptions(); + return new Options(ImmutableList.of( + generalOptions, + buildifierOptions, + new BuildozerOptions(generalOptions, buildifierOptions, workflowOptions), + new FolderDestinationOptions(), + new FolderOriginOptions(), + gitOptions, + new GitOriginOptions(), + new GitHubPrOriginOptions(), + gitDestinationOptions, + new GitHubOptions(generalOptions, gitOptions), + new GitHubDestinationOptions(), + new GerritOptions(generalOptions, gitOptions), + new GitMirrorOptions(generalOptions, gitOptions), + new HgOptions(generalOptions), + new HgOriginOptions(), + new PatchingOptions(generalOptions), + workflowOptions, + new RemoteFileOptions(), + new DebugOptions(generalOptions))); + } + + /** + * A ModuleSet contains the collection of modules and flags for one Skylark copy.bara.sky + * evaluation/execution. + */ + public final ModuleSet create() { + Options options = newOptions(); + return new ModuleSet(options, getStaticModules(), modulesToVariableMap(options)); + } + + private ImmutableMap modulesToVariableMap(Options options) { + return getModules(options).stream() + .collect(ImmutableMap.toImmutableMap( + this::findClosestStarlarkBuiltinName, + Function.identity())); + } + + private String findClosestStarlarkBuiltinName(Object o) { + Class cls = o.getClass(); + while (cls != null && cls != Object.class) { + StarlarkBuiltin annotation = cls.getAnnotation(StarlarkBuiltin.class); + if (annotation != null) { + return annotation.name(); + } + cls = cls.getSuperclass(); + } + throw new IllegalStateException("Cannot find @StarlarkBuiltin for " + o.getClass()); + } +} \ No newline at end of file diff --git a/third_party/copybara/java/com/google/copybara/NonReversibleValidationException.java b/third_party/copybara/java/com/google/copybara/NonReversibleValidationException.java new file mode 100644 index 0000000000..192616c287 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/NonReversibleValidationException.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.devtools.build.lib.syntax.EvalException; +import com.google.devtools.build.lib.syntax.Location; + +/** + * Exception thrown when a {@link Transformation} is not reversible but the configuration asked for + * the reverse. + * + * TODO(malcon): Move to the exception package + */ +public class NonReversibleValidationException extends EvalException { + + public NonReversibleValidationException(Location location, String message) { + super(location, message); + } + + public NonReversibleValidationException(Location location, String message, Throwable cause) { + super(location, message, cause); + } +} diff --git a/third_party/copybara/java/com/google/copybara/Option.java b/third_party/copybara/java/com/google/copybara/Option.java new file mode 100644 index 0000000000..da048df486 --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Option.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +/** + * An {@code Option} indicates a class has options usable by copybara's config system. + */ +public interface Option { +} diff --git a/third_party/copybara/java/com/google/copybara/Options.java b/third_party/copybara/java/com/google/copybara/Options.java new file mode 100644 index 0000000000..79519d4e2b --- /dev/null +++ b/third_party/copybara/java/com/google/copybara/Options.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara; + +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableMap; +import java.util.Map.Entry; + +/** + * A class that groups all the options used in the program + */ +public class Options { + + private final ImmutableMap, Option> config; + + public Options(Iterable options) { + ImmutableMap.Builder, Option> builder = ImmutableMap.builder(); + for (Option option : options) { + builder.put(option.getClass(), option); + } + config = builder.build(); + } + + /** + * Get an option for a given class. + * + * @throws IllegalStateException if the configuration cannot be found + */ + @SuppressWarnings("unchecked") + public T get(Class optionClass) { + Option option = config.get(optionClass); + if (option == null) { + // If we didn't find the exact class, look for a subclass. + for (Entry, Option> entry : config.entrySet()) { + if (optionClass.isAssignableFrom(entry.getKey())) { + return (T) entry.getValue(); + } + } + throw new IllegalStateException("No option type found for " + optionClass); + } + return (T) option; + } + + public ImmutableCollection