diff --git a/.github/actions/generate-catalogs/README.md b/.github/actions/generate-catalogs/README.md new file mode 100644 index 00000000..c553c583 --- /dev/null +++ b/.github/actions/generate-catalogs/README.md @@ -0,0 +1,117 @@ +# Image Catalogs Generator Action + +This composite GitHub Action generates [CloudNativePG ImageCatalogs](https://cloudnative-pg.io/documentation/current/image_catalog/) +from a container registry. +It wraps the [`catalogs_generator.py`](./catalogs_generator.py) script and makes it easy to +run inside CI pipelines. + +--- + +## How it works + +1. The script retrieves all image tags from a container registry. +2. A regular expression is applied to select the tags to include in the ImageCatalog. +3. Matching tags are sorted using [semantic versioning](https://semver.org/). +4. For each PostgreSQL major version, the latest matching tag is chosen. +5. The action generates: + + - One `ClusterImageCatalog` YAML file per requested distribution and image + type + - A `kustomization.yaml` to install/update all cluster catalogs at once + +--- + +## Inputs + +| Name | Required | Description | Example | +| --------------- | --------- | ------------------------------------------------------------------------------- | ----------------------------------- | +| `registry` | ✅ yes | The container registry to query. | `ghcr.io/cloudnative-pg/postgresql` | +| `image-types` | ✅ yes | Comma-separated list of image types. | `minimal,standard` | +| `distributions` | ✅ yes | Comma-separated list of supported OS distributions. | `bookworm,trixie` | +| `regex` | ✅ yes | Regular expression used to match image tags. | *See [Regex](#regex)* | +| `output-dir` | ✅ yes | Directory where generated catalogs will be written. | `./` | +| `family` | ❌ no | Family name for generated catalogs (filename prefix). Defaults to `postgresql`. | `my-custom-family` | + +--- + +## Regex + +The `regex` input defines which tags are added to the `ClusterImageCatalog`. + +- The **first capturing group** must be the PostgreSQL major version: + + - `(\d+)` → e.g. `18` + +- Subsequent capturing groups are optional and may include: + + - an additional version: `(\d+(?:\.\d+)+)` → e.g. `1.2.3` + - a 12 digit timestamp: `(\d{12})` → e.g. `202509161052` + +**Examples:** + +```regex +# Matches '18-202509161052', '18.1-202509161052', etc. +'(\d+)(?:\.\d+|beta\d+|rc\d+|alpha\d+)-(\d{12})' + +# Matches '18-3.0.6-202509161052', '18.1-3.0.6-202509161052', etc. +'(\d+)(?:\.\d+|beta\d+|rc\d+|alpha\d+)-(\d+(?:\.\d+){1,3})-(\d{12})' +``` + +> **Note:** Each `image-types` and `distributions` will be combined together +> to form a suffix, `--`, which will internally be +> appended to the `regex` provided. Tags that do not contain explicit +> image type and distribution as a suffix are currently not supported. + +--- + +### Family + +The optional `family` input customizes: + +1. **File prefix**: `-minimal-trixie.yaml` +2. **`metadata.name`** in the ImageCatalog: `-minimal-trixie` +3. **`images.cnpg.io/family` label** on the ImageCatalog object + +If not specified it defaults to `postgresql`. + +--- + +## Usage + +Example workflow: + +``` +jobs: + generate-catalogs: + runs-on: ubuntu-latest + steps: + - name: Generate image catalogs + uses: cloudnative-pg/postgres-containers/.github/actions/generate-catalogs@main + with: + registry: ghcr.io/cloudnative-pg/postgresql + image-types: minimal,standard + distributions: bookworm,trixie + regex: '(\d+)(?:\.\d+|beta\d+|rc\d+|alpha\d+)-(\d{12})' + output-dir: . +``` + +This generates: + +``` +./catalog-minimal-bookworm.yaml +./catalog-standard-bookworm.yaml +./catalog-minimal-trixie.yaml +./catalog-standard-trixie.yaml +``` + +The generated `kustomization.yaml` will look like: + +``` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- catalog-minimal-bookworm.yaml +- catalog-standard-bookworm.yaml +- catalog-minimal-trixie.yaml +- catalog-standard-trixie.yaml +``` diff --git a/.github/actions/generate-catalogs/action.yml b/.github/actions/generate-catalogs/action.yml new file mode 100644 index 00000000..57ace6db --- /dev/null +++ b/.github/actions/generate-catalogs/action.yml @@ -0,0 +1,74 @@ +name: Generate Image Catalogs +description: Generate Image Catalogs +inputs: + registry: + description: "The registry to interrogate" + required: true + image-types: + description: "Image types to retrieve - comma separated values" + required: true + distributions: + description: "OS distributions to retrieve - comma separated values" + required: true + regex: + description: "The regular expression used to retrieve container images" + required: true + output-dir: + description: "The path to output directory" + required: true + family: + description: "The family name to assign to the catalogs" + required: false + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6 + with: + python-version: 3.13 + + - name: Install Python dependencies + shell: bash + run: | + pip install packaging==25.0 PyYAML==6.0.2 + + - name: Generate catalogs + shell: bash + env: + REGISTRY: ${{ inputs.registry }} + IMAGE_TYPES: ${{ inputs.image-types }} + DISTRIBUTIONS: ${{ inputs.distributions }} + REGEX: ${{ inputs.regex }} + OUTPUT_DIR: ${{ inputs.output-dir }} + FAMILY: ${{ inputs.family }} + run: | + set -euo pipefail + ARGS=() + + if [[ -n "${REGISTRY:-}" ]]; then + ARGS+=( --registry "$REGISTRY" ) + fi + + if [[ -n "${IMAGE_TYPES:-}" ]]; then + IFS=',' read -r -a image_types <<< "$IMAGE_TYPES" + ARGS+=( --image-types "${image_types[@]}" ) + fi + + if [[ -n "${DISTRIBUTIONS:-}" ]]; then + IFS=',' read -r -a distributions <<< "$DISTRIBUTIONS" + ARGS+=( --distributions "${distributions[@]}" ) + fi + + if [[ -n "${REGEX:-}" ]]; then + ARGS+=( --regex "$REGEX" ) + fi + + if [[ -n "${FAMILY:-}" ]]; then + ARGS+=( --family "$FAMILY" ) + fi + + ARGS+=( --output-dir "$OUTPUT_DIR" ) + + echo "Running: python $GITHUB_ACTION_PATH/catalogs_generator.py ${ARGS[*]}" + python "$GITHUB_ACTION_PATH/catalogs_generator.py" "${ARGS[@]}" diff --git a/.github/catalogs_generator.py b/.github/actions/generate-catalogs/catalogs_generator.py similarity index 56% rename from .github/catalogs_generator.py rename to .github/actions/generate-catalogs/catalogs_generator.py index f5f68763..b6e643b4 100644 --- a/.github/catalogs_generator.py +++ b/.github/actions/generate-catalogs/catalogs_generator.py @@ -31,11 +31,50 @@ supported_img_types = ["minimal", "standard", "system"] supported_os_names = ["bullseye", "bookworm", "trixie"] min_supported_major = 13 -repo_name = "cloudnative-pg/postgresql" -full_repo_name = f"ghcr.io/{repo_name}" -pg_regexp = r"(\d+)(?:\.\d+|beta\d+|rc\d+|alpha\d+)-(\d{12})" +default_registry = "ghcr.io/cloudnative-pg/postgresql" +default_family = "postgresql" +default_regex = r"(\d+)(?:\.\d+|beta\d+|rc\d+|alpha\d+)-(\d{12})" _token_cache = {"value": None, "expires_at": 0} +normalized_pattern = re.compile( + r""" + ^(?P\d+(?:\.\d+|beta\d+|rc\d+|alpha\d+)) # A mandatory PostgreSQL version + (?:-(?P\d+(?:\.\d+)+))? # An optional extension version + (?:-(?P\d{12}))? # An optional timestamp + $ + """, + re.VERBOSE, +) + + +# Normalize a tag to make it a valid PEP 440 version. +# Optional capture groups after the Postgres version will +# be appended using a "+"" as a local version segment, and +# concatenated with "." in case there's more then one. +def normalize_tag(tag): + match = normalized_pattern.match(tag) + if not match: + raise ValueError(f"Unrecognized tag format: {tag}") + + pg_version = match.group("pg_version") + extension_version = match.group("extension_version") + timestamp = match.group("timestamp") + + # Build PEP 440 compliant version + # e.g 17.6, 17.6+202509161052, 17.6+3.6.0.202509161052 + extra_match = [] + if extension_version: + extra_match.append(extension_version) + if timestamp: + extra_match.append(timestamp) + + if extra_match: + normalized_tag = f"{pg_version}+{'.'.join(extra_match)}" + else: + normalized_tag = pg_version + + return version.Version(normalized_tag) + def get_json(image_name): data = check_output( @@ -52,14 +91,14 @@ def get_json(image_name): return repo_json -def get_token(repository_name): +def get_token(image_name): global _token_cache now = time.time() if _token_cache["value"] and now < _token_cache["expires_at"]: return _token_cache["value"] - url = "https://ghcr.io/token?scope=repository:{}:pull".format(repository_name) + url = "https://ghcr.io/token?scope=repository:{}:pull".format(image_name) with urllib.request.urlopen(url) as response: data = json.load(response) token = data["token"] @@ -70,13 +109,14 @@ def get_token(repository_name): def get_digest(repository_name, tag): - token = get_token(repository_name) + image_name = repository_name.removeprefix("ghcr.io/") + token = get_token(image_name) media_types = [ "application/vnd.oci.image.index.v1+json", "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json", ] - url = f"https://ghcr.io/v2/{repository_name}/manifests/{tag}" + url = f"https://ghcr.io/v2/{image_name}/manifests/{tag}" req = urllib.request.Request(url) req.add_header("Authorization", "Bearer {}".format(token)) req.add_header("Accept", ",".join(media_types)) @@ -85,6 +125,14 @@ def get_digest(repository_name, tag): return digest +def get_filename(family, img_type, os_name): + filename_prefix = "catalog" + if family != default_family: + filename_prefix = family + + return f"{filename_prefix}-{img_type}-{os_name}.yaml" + + def write_catalog(tags, version_re, img_type, os_name, output_dir="."): image_suffix = f"-{img_type}-{os_name}" version_re = re.compile(rf"^{version_re}{re.escape(image_suffix)}$") @@ -97,7 +145,7 @@ def write_catalog(tags, version_re, img_type, os_name, output_dir="."): tags = [item for item in tags if not exclude_preview.search(item)] # Sort the tags according to semantic versioning - tags.sort(key=lambda v: version.Version(v.removesuffix(image_suffix)), reverse=True) + tags.sort(key=lambda v: normalize_tag(v.removesuffix(image_suffix)), reverse=True) results = {} for item in tags: @@ -112,16 +160,19 @@ def write_catalog(tags, version_re, img_type, os_name, output_dir="."): continue if major not in results: - digest = get_digest(repo_name, item) - results[major] = [f"{full_repo_name}:{item}@{digest}"] + digest = get_digest(args.registry, item) + results[major] = [f"{args.registry}:{item}@{digest}"] + + if not results: + raise RuntimeError("No results have been found!") catalog = { "apiVersion": "postgresql.cnpg.io/v1", "kind": "ClusterImageCatalog", "metadata": { - "name": f"postgresql{image_suffix}", + "name": f"{args.family}{image_suffix}", "labels": { - "images.cnpg.io/family": "postgresql", + "images.cnpg.io/family": args.family, "images.cnpg.io/type": img_type, "images.cnpg.io/os": os_name, "images.cnpg.io/date": time.strftime("%Y%m%d"), @@ -137,7 +188,7 @@ def write_catalog(tags, version_re, img_type, os_name, output_dir="."): } os.makedirs(output_dir, exist_ok=True) - output_file = os.path.join(output_dir, f"catalog{image_suffix}.yaml") + output_file = os.path.join(output_dir, get_filename(args.family, img_type, os_name)) with open(output_file, "w") as f: yaml.dump(catalog, f, sort_keys=False) @@ -146,20 +197,47 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description="CloudNativePG ClusterImageCatalog YAML generator" ) + parser.add_argument( + "--registry", + default=default_registry, + help=f"The registry to interrogate (default: {default_registry})", + ) parser.add_argument( "--output-dir", default=".", help="Directory to save the YAML files" ) + parser.add_argument( + "--regex", + default=default_regex, + help=f"The regular expression used to retrieve container image. The first capturing group must be the PostgreSQL major version. (default: {default_regex})", + ) + parser.add_argument( + "--image-types", + nargs="+", + default=supported_img_types, + help=f"Image types to retrieve (default: {supported_img_types})", + ) + parser.add_argument( + "--distributions", + nargs="+", + default=supported_os_names, + help=f"Distributions to retrieve (default: {supported_os_names})", + ) + parser.add_argument( + "--family", + default=default_family, + help=f"The family name to assign to the catalogs (default: {default_family})", + ) args = parser.parse_args() - repo_json = get_json(full_repo_name) + repo_json = get_json(args.registry) tags = repo_json["Tags"] catalogs = [] - for img_type in supported_img_types: - for os_name in supported_os_names: - filename = f"catalog-{img_type}-{os_name}.yaml" + for img_type in args.image_types: + for os_name in args.distributions: + filename = get_filename(args.family, img_type, os_name) print(f"Generating {filename}") - write_catalog(tags, pg_regexp, img_type, os_name, args.output_dir) + write_catalog(tags, args.regex, img_type, os_name, args.output_dir) catalogs.append(filename) kustomization = { diff --git a/.github/workflows/catalogs.yml b/.github/workflows/catalogs.yml index f5a61769..a10bc436 100644 --- a/.github/workflows/catalogs.yml +++ b/.github/workflows/catalogs.yml @@ -15,6 +15,7 @@ jobs: update-catalogs: runs-on: ubuntu-24.04 steps: + # TODO: remove this step once system images are EOL - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 with: @@ -29,18 +30,15 @@ jobs: token: ${{ secrets.REPO_GHA_PAT }} ref: main - - name: Set up Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6 - with: - python-version: 3.13 - - - name: Install Python dependencies - run: | - pip install packaging==25.0 PyYAML==6.0.2 - - name: Generate catalogs - run: | - python postgres-containers/.github/catalogs_generator.py --output-dir artifacts/image-catalogs/ + uses: ./postgres-containers/.github/actions/generate-catalogs + with: + output-dir: artifacts/image-catalogs/ + registry: ghcr.io/cloudnative-pg/postgresql + family: postgresql + distributions: bullseye,bookworm,trixie + image-types: minimal,standard,system + regex: '(\d+)(?:\.\d+|beta\d+|rc\d+|alpha\d+)-(\d{12})' # TODO: remove this step once system images are EOL - name: Update legacy catalogs