feat: conformance reports persist

.github/workflows/rpc-parity-report.yml

@@ -0,0 +1,65 @@
+name: RPC parity conformance report
+on:
+  workflow_dispatch:
+    inputs:
+      image:
+        description: "Forest Docker image to test against"
+        required: false
+        default: "ghcr.io/chainsafe/forest:edge-fat"
+  schedule:
+    - cron: "0 0 1 * *" # Monthly, on the first day of the month at midnight
+jobs:
+  rpc-parity:
+    name: RPC parity report
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - name: Relocate docker volumes folder
+        run: |
+          # move docker volumes folder to under `/mnt` which has 60GB+ free space
+          sudo ls /var/lib/docker/volumes
+          sudo mv /var/lib/docker/volumes /mnt/docker-volumes
+          sudo ln -sf /mnt/docker-volumes /var/lib/docker/volumes
+          sudo ls /var/lib/docker/volumes
+          df -h
+      - uses: actions/checkout@v6
+      - name: Run api compare tests on calibnet
+        shell: bash
+        run: |
+          IMAGE=${{ github.event.inputs.image }}
+          if [ -z "$IMAGE" ]; then
+            IMAGE="ghcr.io/chainsafe/forest:edge-fat"
+          fi
+          echo "FROM $IMAGE" > Dockerfile-RPC
+          export FOREST_DOCKERFILE_OVERRIDE=Dockerfile-RPC
+          ./scripts/tests/api_compare/setup.sh
+        timeout-minutes: 30
+      - name: Dump docker logs
+        if: always()
+        uses: jwalton/gh-docker-logs@v2
+      - name: Dump Docker usage
+        if: always()
+        run: |
+          sudo ls /var/lib/docker/volumes
+          docker system df
+          docker system df --verbose
+          df -h
+      - uses: jdx/mise-action@v3
+      - run: mise add_conformance_report
+      # This is needed in order to have the commits signed.
+      - uses: actions/create-github-app-token@v2
+        id: generate-token
+        with:
+          app-id: ${{ secrets.LESHY_APP_ID }}
+          private-key: ${{ secrets.LESHY_APP_PRIVATE_KEY }}
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v8
+        with:
+          base: main
+          branch: leshy/scheduled-rpc-parity-report
+          token: ${{ steps.generate-token.outputs.token }}
+          commit-message: Add monthly RPC parity conformance report
+          sign-commits: true
+          title: "[automated] Add monthly RPC parity conformance report"
+          body: |
+            ### Changes
+            - Adds the monthly RPC parity conformance report to the docs.

docs/docs/users/reports/_category_.json

@@ -0,0 +1,4 @@
+{
+  "label": "Reports",
+  "position": 5
+}

docs/docs/users/reports/api_conformance/_category_.json

@@ -0,0 +1,4 @@
+{
+  "label": "API Conformance Reports",
+  "position": 1
+}

mise-tasks/add_conformance_report.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+#MISE description="Extracts conformance report from the API compare compose volume and adds it to the docs."
+
+TMP_REPORT=$(mktemp -d)/report.json
+DATE=$(date +%Y-%m-%d)
+./scripts/tests/api_compare/extract_conformance_report.sh "$TMP_REPORT"
+./scripts/tests/api_compare/convert_report_to_markdown.sh "$TMP_REPORT" ./docs/docs/users/reports/api_conformance/report_"$DATE".md

scripts/tests/api_compare/convert_report_to_markdown.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Simple script to generate a markdown report from test.json
+# Usage: ./convert_report_to_markdown.sh <input_json> <output_markdown>
+
+set -euo pipefail
+
+INPUT_FILE="$1"
+OUTPUT_FILE="$2"
+
+# Check if input file exists
+if [[ ! -f "$INPUT_FILE" ]]; then
+    echo "Error: Input file '$INPUT_FILE' not found."
+    exit 1
+fi
+
+# Generate the markdown report
+{
+    echo "# $(date '+%Y-%m-%d') - API Parity Report"
+    echo ""
+    echo "| Method | Lotus-conformance check |"
+    echo "|--------|-------------------------|"
+
+    jq -r '.methods[] | 
+        if (.name | startswith("Forest.")) then
+            "| `\(.name)` | N/A (Forest-specific) |"
+        else
+            "| `\(.name)` | \(if .status.type == "tested" then "✅" else "❌" end) |"
+        end' "$INPUT_FILE"
+
+} > "$OUTPUT_FILE"
+
+echo "Report generated: $OUTPUT_FILE"

scripts/tests/api_compare/extract_conformance_report.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# This script extracts the conformance report from the finished compose volume.
+# Usage: ./extract_conformance_report.sh <output_path>
+
+set -euo pipefail
+
+OUTPUT_PATH="$1"
+
+# Volume is in the `node-data` volume
+VOLUME_NAME="api_compare_node-data"
+TEMP_CONTAINER_NAME="temp-extract-container"
+
+# Create a temporary container to access the volume
+docker run --name "$TEMP_CONTAINER_NAME" -v "$VOLUME_NAME":/data:ro -d alpine sleep infinity
+trap 'docker rm -f "$TEMP_CONTAINER_NAME"' EXIT
+
+REPORT_PATH_IN_VOLUME=$(docker exec "$TEMP_CONTAINER_NAME" sh -c 'find /data/api-compare-report -name "*.json" | head -n 1')
+
+# Ensure the report file was found
+if [ -z "$REPORT_PATH_IN_VOLUME" ]; then
+  echo "Conformance report not found in the volume."
+  exit 1
+fi
+
+# Copy the conformance report from the volume to the specified output path
+docker cp "$TEMP_CONTAINER_NAME:$REPORT_PATH_IN_VOLUME" "$OUTPUT_PATH"