Migrate CI pipeline from java-profiler-build into java-profiler

.adms/python/gitlab.yaml

@@ -0,0 +1,9 @@
+# File generated and managed by #dependency-management.
+# Changes are subject to overwriting.
+# DO NOT EDIT
+
+variables:
+    PIP_INDEX_URL: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple"
+    PIP_EXTRA_INDEX_URL: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/testing/@current/simple"
+    UV_INDEX: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple https://depot-read-api-python.us1.ddbuild.io/magicmirror/testing/@current/simple"
+    UV_DEFAULT_INDEX: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple"

.github/chainguard/async-profiler-build.ci.sts.yaml

@@ -1,10 +1,10 @@
 # Allow async-profiler-build CI to publish to gh-pages
 issuer: https://gitlab.ddbuild.io
 
-subject_pattern: "project_path:DataDog/apm-reliability/async-profiler-build:ref_type:branch:ref:.*"
+subject_pattern: "project_path:DataDog/java-profiler:ref_type:branch:ref:.*"
 
 claim_pattern:
-  project_path: "DataDog/apm-reliability/async-profiler-build"
+  project_path: "DataDog/java-profiler"
   ref_type: "branch"
   ref: ".*"
 

.github/chainguard/gh-pages.sts.yaml

@@ -0,0 +1,26 @@
+# Octo-STS Trust Policy for GitHub Pages Publishing
+# This policy allows GitLab CI to push integration test reports to gh-pages branch
+#
+# Trust Policy Location: .github/chainguard/gh-pages.sts.yaml
+# Referenced by: scripts/get-github-token-via-octo-sts.sh (OCTO_STS_POLICY=gh-pages)
+#
+# How it works:
+# 1. GitLab CI generates OIDC token with issuer: https://gitlab.ddbuild.io
+# 2. Token includes claims: project_path, ref, namespace_path, etc.
+# 3. Octo-STS validates token against this policy
+# 4. If valid, Octo-STS returns short-lived GitHub token with specified permissions
+
+# GitLab OIDC issuer
+issuer: https://gitlab.ddbuild.io
+
+# Match GitLab CI jobs from any branch (needed for PR comments)
+# GitLab token includes: project_path=DataDog/java-profiler, ref=<branch>
+subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.*
+
+# GitHub API permissions for the returned token
+# contents:write - Required to push to gh-pages branch
+permissions:
+  contents: write
+
+# Token lifetime (default: 1 hour)
+# Short-lived tokens reduce security risk

.github/chainguard/update-images.sts.yaml

@@ -0,0 +1,17 @@
+# Octo-STS Trust Policy for Image Update PRs
+#
+# Allows the GitLab CI check-image-updates and rebuild-images-pr jobs to push
+# branches and create pull requests for CI image reference updates.
+#
+# Referenced by: scripts/create-image-update-pr.sh (OCTO_STS_POLICY=update-images)
+
+# GitLab OIDC issuer
+issuer: https://gitlab.ddbuild.io
+
+# Match GitLab CI jobs from the async-profiler-build project on any branch
+subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.*
+
+# GitHub API permissions
+permissions:
+  contents: write
+  pull_requests: write

.gitignore

@@ -2,6 +2,7 @@
 **/build_*/
 **/build-*/
 !build-logic/
+!.gitlab/build-deploy/
 /nbproject/
 /out/
 /.idea/

.gitlab-ci.yml

@@ -1,24 +1,159 @@
-# Triggers a build within the Datadog infrastructure in the ddprof-build repository
-trigger_internal_build:
+image: alpine
+
+variables:
+  REGISTRY: registry.ddbuild.io
+  PREPARE_IMAGE: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest
+  # Image with dd-octo-sts for GitHub token exchange (check-image-updates, rebuild-images-pr)
+  DD_OCTO_STS_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
+  FORCE_BUILD:
+    value: ""
+    description: "Force build even if no new commits (any non-empty value)"
+
+default:
+  tags: ["arch:amd64"]
+  interruptible: true
+  before_script:
+    - '[ "${CANCELLED:-}" != "true" ] || { echo "No PR for this branch — skipping job"; exit 0; }'
+
+stages:
+  - images
+  - generate-signing-key
+  - prepare
+  - build
+  - stresstest
+  - deploy
+  - integration-test
+  - reliability
+  - benchmarks
+  - notify
+
+# Detects newer images in registry and creates GitHub PR with updates
+check-image-updates:
+  stage: images
   rules:
-    - if: $CI_COMMIT_BRANCH =~ /release\/.*/
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CHECK_IMAGE_UPDATES == "true"'
+      when: always
+    - if: '$CI_PIPELINE_SOURCE == "web"'
+      when: manual
+      allow_failure: true
+  extends: .bootstrap-gh-tools
+  tags: ["arch:arm64"]
+  image: ${DD_OCTO_STS_IMAGE}
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
+  script:
+    - set -euo pipefail
+    - echo "Checking for image updates..."
+    - .gitlab/scripts/check-image-updates.sh > updates.json
+    - |
+      update_count=$(jq 'length' updates.json)
+      echo "Found ${update_count} update(s)"
+      if [ "$update_count" -gt 0 ]; then
+        echo "Updates available:"
+        jq . updates.json
+        .gitlab/scripts/create-image-update-pr.sh updates.json
+      else
+        echo "All images are up to date"
+      fi
+  artifacts:
+    when: always
+    paths:
+      - updates.json
+    expire_in: 7 days
+
+rebuild-images:
+  stage: images
+  rules:
+    - if: '$CI_COMMIT_TAG'
       when: never
-    - when: always
-      allow_failure: false
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - when: manual
+      allow_failure: true
+  tags: ["arch:amd64"]
+  variables:
+    REBUILD_IMAGES: ""   # comma/space-separated short names, or empty = all
+  image: ${DOCKER_IMAGE}
+  id_tokens:
+    DDSIGN_ID_TOKEN:
+      aud: image-integrity
+  script:
+    - set -euo pipefail
+    - .gitlab/scripts/rebuild-images.sh
+  artifacts:
+    when: always
+    paths:
+      - updates.json
+    expire_in: 1 day
+
+rebuild-images-pr:
+  stage: images
+  rules:
+    - if: '$CI_COMMIT_TAG'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - when: on_success
+  needs:
+    - job: rebuild-images
+      artifacts: true
+  extends: .bootstrap-gh-tools
+  tags: ["arch:arm64"]
+  image: ${DD_OCTO_STS_IMAGE}
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
+  script:
+    - set -euo pipefail
+    - .gitlab/scripts/create-image-update-pr.sh updates.json
+
+create_key:
+  stage: generate-signing-key
+  when: manual
+  needs: []
+  tags: ["arch:amd64"]
   variables:
-    DOWNSTREAM_BRANCH: "main"
-    UPSTREAM_PROJECT: ${CI_PROJECT_PATH}
-    UPSTREAM_PROJECT_NAME: ${CI_PROJECT_NAME}
-    UPSTREAM_BRANCH: ${CI_COMMIT_BRANCH}
-    UPSTREAM_COMMIT_SHA: ${CI_COMMIT_SHA}
-    DDPROF_DEFAULT_BRANCH: "main"
-    DDPROF_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
-    DDROF_COMMIT_SHA: ${CI_COMMIT_SHA}
-    DPROF_SHORT_COMMIT_SHA: ${CI_COMMIT_SHORT_SHA}
-    DDPROF_COMMIT_TAG: ${CI_COMMIT_TAG}
+    PROJECT_NAME: "java-profiler"
+    EXPORT_TO_KEYSERVER: "true"
+    KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: java-profiler
+  image: $REGISTRY/ci/agent-key-management-tools/gpg:1
+  script:
+    - /create.sh
+  artifacts:
+    expire_in: 13 mos
+    paths:
+      - pubkeys
+
+# Shared version detection used by benchmarks and reliability pipelines
+get-versions:
+  extends: .get-versions
+  needs:
+    - job: prepare:start
+      artifacts: false
+
+# Triggered externally from async-profiler-build with JDK build parameters;
+# kept as a child pipeline because it is mutually exclusive with the main build
+jdk-integration-test:
+  stage: build
+  rules:
+    - if: '$JDK_VERSION == null || $DEBUG_LEVEL == null || $HASH == null || $DOWNSTREAM == null'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "web"'
+      when: always
+      allow_failure: false
+    - when: always
   trigger:
-    project: DataDog/apm-reliability/async-profiler-build
+    include: .gitlab/jdk-integration/.gitlab-ci.yml
     strategy: depend
-    branch: $DOWNSTREAM_BRANCH
     forward:
       pipeline_variables: true
+
+include:
+  - local: .gitlab/common.yml
+  - local: .adms/python/gitlab.yaml
+  - local: .gitlab/benchmarks/images.yml
+  - local: .gitlab/build-deploy/images.yml
+  - local: .gitlab/build-deploy/.gitlab-ci.yml
+  - local: .gitlab/benchmarks/.gitlab-ci.yml
+  - local: .gitlab/reliability/.gitlab-ci.yml

.gitlab/Dockerfile.datadog-ci

@@ -0,0 +1,60 @@
+ARG BASEIMAGE=registry.ddbuild.io/images/base/gbi-ubuntu_2404:release
+FROM ${BASEIMAGE}
+
+USER root
+
+# Create non-root user for security
+RUN useradd --create-home --shell /bin/bash --uid 1001 ci-user
+
+# Install Node.js 20 and npm
+# Default seems to be 14 which does not work with datadog-ci
+RUN set -x \
+    && apt-get update && apt-get -y install --no-install-recommends curl xz-utils\
+    && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs \
+    pipx=1.4.3-1 \
+    binutils \
+    jq \
+    && npm install -g @datadog/datadog-ci@3.16.0 \
+    && apt-get -y clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install GitHub CLI
+RUN set -x \
+    && curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
+        -o /usr/share/keyrings/githubcli-archive-keyring.gpg \
+    && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
+    && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \
+        > /etc/apt/sources.list.d/github-cli.list \
+    && apt-get update \
+    && apt-get install -y gh \
+    && apt-get -y clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# awscli is not available in Ubuntu 2404 for some inexplicable reason so lets install in via other means
+RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install awscli
+
+# Install Go 1.22.3
+RUN set -x \
+    && curl -LO https://golang.org/dl/go1.22.3.linux-amd64.tar.gz \
+    && tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz \
+    && rm go1.22.3.linux-amd64.tar.gz
+
+# Set up Go environment for root and install Crane
+ENV PATH="/usr/local/go/bin:${PATH}"
+ENV GOPATH="/root/go"
+ENV GOBIN="/usr/local/bin"
+
+# Install Crane version 0.19.1 directly to /usr/local/bin so it's available for all users
+RUN set -x \
+    && go install github.com/google/go-containerregistry/cmd/crane@v0.19.1
+
+# Switch to non-root user
+USER ci-user
+WORKDIR /home/ci-user
+
+# Set PATH for the ci-user (crane is now in /usr/local/bin)
+ENV PATH="/usr/local/go/bin:/usr/local/bin:${PATH}"
+
+# Verify installation (as non-root user)
+RUN node -v && npm -v && go version && crane version && datadog-ci --help && jq --version && gh --version

.gitlab/base/Dockerfile

@@ -0,0 +1,10 @@
+ARG BASE_IMAGE=openjdk:11-slim-buster
+FROM ${BASE_IMAGE} as base
+ARG CI_JOB_TOKEN
+WORKDIR /root
+
+RUN mkdir -p /usr/share/man/man1 # https://github.com/debuerreotype/docker-debian-artifacts/issues/24
+RUN (apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends curl git moreutils awscli amazon-ecr-credential-helper gnupg2 npm build-essential wget bsdmainutils clang libclang-rt-dev jq zip unzip maven) || true
+RUN (apk update && apk add curl git moreutils aws-cli docker-credential-ecr-login gnupg alpine-sdk build-base wget npm hexdump linux-headers clang compiler-rt bash jq gradle zip unzip) || true
+RUN npm install -g --save-dev @datadog/datadog-ci
+RUN  rm -rf "/var/lib/apt/lists/*"

.gitlab/base/centos7/Dockerfile

@@ -0,0 +1,43 @@
+ARG BASE_IMAGE=openjdk:11-slim-buster
+FROM ${BASE_IMAGE} as base
+ARG CI_JOB_TOKEN
+WORKDIR /root
+
+# 1. Replace dead mirrorlist entries with HTTPS vault URLs
+RUN set -eux; \
+    sed -i -e 's/^mirrorlist/#mirrorlist/' \
+           -e 's|^#baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|' \
+           /etc/yum.repos.d/CentOS-*.repo
+
+# 2. Add a vault mirror that still contains Software Collections
+RUN cat > /etc/yum.repos.d/CentOS-SCLo-Vault.repo <<'EOF'
+[centos-sclo-rh]
+name=CentOS-7 - SCLo rh (Rocky Vault)
+baseurl=https://dl.rockylinux.org/vault/centos/7.9.2009/sclo/$basearch/rh/
+gpgcheck=0
+enabled=1
+
+[centos-sclo-sclo]
+name=CentOS-7 - SCLo sclo (Rocky Vault)
+baseurl=https://dl.rockylinux.org/vault/centos/7.9.2009/sclo/$basearch/sclo/
+gpgcheck=0
+enabled=1
+EOF
+
+# 3. Expose devtoolset-11 binaries & libs by default (they are installed a bit later)
+ENV PATH="/opt/rh/devtoolset-11/root/usr/bin:${PATH}" \
+    LD_LIBRARY_PATH="/opt/rh/devtoolset-11/root/usr/lib64:${LD_LIBRARY_PATH}"
+
+RUN yum -y clean all
+RUN yum -y update && yum -y install scl-utils devtoolset-11 devtoolset-11-toolchain curl zip unzip git libstdc++-static make which wget cmake binutils
+RUN yum -y clean all
+RUN (curl -s "https://get.sdkman.io" | bash)
+RUN (source ~/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem)
+RUN (curl -sL https://rpm.nodesource.com/setup_16.x | bash -)
+# installing JQ requires two steps - adding the repo and then installing the tool
+RUN yum install -y epel-release
+RUN yum install -y jq
+# now install nodejs and datadog CI support
+RUN yum -y install nodejs
+RUN npm install -g --save-dev @datadog/datadog-ci
+RUN  rm -rf "/var/lib/apt/lists/*"