diff --git a/.github/PULL_REQUEST_TEMPLATE.MD b/.github/PULL_REQUEST_TEMPLATE.MD index f0244408..4fc5795f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.MD +++ b/.github/PULL_REQUEST_TEMPLATE.MD @@ -12,6 +12,14 @@ Only one reference is required - either GitHub issue OR ADO Work Item. > GitHub Issue: # + + ------------------------------------------------------------------- ### Summary diff --git a/docs/plan-crossRepoDevBcp.prompt.md b/docs/plan-crossRepoDevBcp.prompt.md new file mode 100644 index 00000000..4c8d9e59 --- /dev/null +++ b/docs/plan-crossRepoDevBcp.prompt.md @@ -0,0 +1,58 @@ +## Plan: Cross-Repo Development for mssql-python + mssql-tds BCP + +Enable parallel development across `mssql-python` (GitHub, `public` ADO project) and `mssql-tds` (ADO repo, `mssql-rs` project) with linked PRs for BCP feature implementation using Rust bindings. + +### Steps + +1. **Add artifact publishing to `mssql-tds` PR pipeline**: Modify [validation-pipeline.yml](../../mssql-tds/.pipeline/validation-pipeline.yml) to publish `mssql_py_core` `.so` files as named pipeline artifacts with PR metadata, e.g., `mssql-py-core-$(Build.BuildId)`. Ensure artifacts are published for PR builds (currently may be filtered). + +2. **Create cross-repo artifact download in `mssql-python` PR pipeline**: Add a conditional stage to [pr-validation-pipeline.yml](../eng/pipelines/pr-validation-pipeline.yml) that: parses PR description for `Depends-On: mssql-tds#`, uses `DownloadPipelineArtifact@2` with `project: mssql-rs` to fetch `.so` artifacts for all platforms (Linux/macOS/Alpine), places them in `mssql_python/` folder. Skip BCP tests if no `Depends-On` found. + +3. **Verify cross-project service connection**: Confirm `Public Artifact Access` connection in `public` project can access `mssql-rs` pipelines. If not, create via Project Settings → Service Connections → Azure DevOps pointing to `mssql-rs`. + +4. **Update PR template with Depends-On section**: Add commented section to [PULL_REQUEST_TEMPLATE.MD](../.github/PULL_REQUEST_TEMPLATE.MD) for BCP developers to specify cross-repo dependency. Add after the GitHub Issue section: + ```markdown + + ``` + +5. **Create local dev script `scripts/build-py-core.sh`**: Add to `mssql-python` repo—expects `mssql-tds` as sibling folder, runs `maturin develop` or `maturin build` in `../mssql-tds/mssql-py-core/`, copies `.so` to `mssql_python/` with platform-appropriate naming. + +6. **Create devcontainer for dual-repo development**: Add `.devcontainer/` to `mssql-python` with Rust + Python + maturin, expects both repos mounted as siblings, auto-runs `build-py-core.sh` on start. + +7. **Document the workflow**: Add section to `CONTRIBUTING.md` or `docs/cross-repo-dev.md` explaining: sibling folder structure, PR template usage, local dev script. + +### Folder Structure (Local Development) + +``` +parent-folder/ +├── mssql-python/ # GitHub repo +└── mssql-tds/ # ADO repo (cloned alongside) +``` + +### CI Flow Diagram + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ mssql-tds repo (mssql-rs project) │ +│ │ +│ PR #123: "Add BCP bindings" │ +│ └──► Pipeline builds mssql_py_core .so (all platforms) │ +│ └──► Publishes artifact: mssql-py-core-{BuildId} │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ mssql-python repo (public project) │ +│ │ +│ PR #456: "Add Cursor.bulkcopy() API" │ +│ │ PR Description: "Depends-On: mssql-tds#123" │ +│ │ │ +│ └──► Pipeline parses PR desc → extracts mssql-tds#123 │ +│ └──► Downloads all platform .so artifacts │ +│ └──► Runs tests on each platform with .so │ +└─────────────────────────────────────────────────────────────────┘ +``` diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 5503e501..e9bcdb36 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -434,6 +434,12 @@ jobs: sqlVersion: 'SQL2025' steps: + # Download mssql-py-core artifacts from mssql-tds if Depends-On is specified in PR + - template: templates/download-mssql-tds-artifacts.yml + parameters: + osType: 'MacOS' + architecture: 'x64' + - task: UsePythonVersion@0 inputs: versionSpec: '3.13' @@ -546,6 +552,12 @@ jobs: useAzureSQL: 'false' steps: + # Download mssql-py-core artifacts from mssql-tds if Depends-On is specified in PR + - template: templates/download-mssql-tds-artifacts.yml + parameters: + osType: 'Linux' + architecture: 'x64' + - script: | # Create a Docker container for testing docker run -d --name test-container-$(distroName) \ @@ -846,6 +858,12 @@ jobs: archName: 'arm64' steps: + # Download mssql-py-core artifacts from mssql-tds if Depends-On is specified in PR + - template: templates/download-mssql-tds-artifacts.yml + parameters: + osType: 'Linux' + architecture: 'ARM64' + - script: | # Set up Docker buildx for multi-architecture support docker run --rm --privileged multiarch/qemu-user-static --reset -p yes @@ -1474,6 +1492,12 @@ jobs: vmImage: 'ubuntu-latest' steps: + # Download mssql-py-core artifacts from mssql-tds if Depends-On is specified in PR + - template: templates/download-mssql-tds-artifacts.yml + parameters: + osType: 'Alpine' + architecture: 'x64' + - script: | # Set up Docker buildx for multi-architecture support docker run --rm --privileged multiarch/qemu-user-static --reset -p yes @@ -1716,6 +1740,12 @@ jobs: vmImage: 'ubuntu-latest' steps: + # Download mssql-py-core artifacts from mssql-tds if Depends-On is specified in PR + - template: templates/download-mssql-tds-artifacts.yml + parameters: + osType: 'Alpine' + architecture: 'ARM64' + - script: | # Set up Docker buildx for multi-architecture support docker run --rm --privileged multiarch/qemu-user-static --reset -p yes diff --git a/eng/pipelines/templates/download-mssql-tds-artifacts.yml b/eng/pipelines/templates/download-mssql-tds-artifacts.yml new file mode 100644 index 00000000..07915e42 --- /dev/null +++ b/eng/pipelines/templates/download-mssql-tds-artifacts.yml @@ -0,0 +1,112 @@ +# Template to download mssql-py-core artifacts from mssql-tds pipeline +# Used for cross-repo BCP development - downloads .so/.whl files built by mssql-tds +# +# Usage in PR description: +# Depends-On: mssql-tds# +# +# Parameters: +# osType: Linux, Alpine, MacOS +# architecture: x64, ARM64 + +parameters: +- name: osType + type: string + default: 'Linux' + values: + - Linux + - Alpine + - MacOS +- name: architecture + type: string + default: 'x64' + values: + - x64 + - ARM64 + +steps: +# Parse PR description for Depends-On: mssql-tds# +- bash: | + echo "Checking for cross-repo dependency in PR description..." + + if [ "$(Build.Reason)" = "PullRequest" ]; then + PR_NUMBER=$(System.PullRequest.PullRequestNumber) + echo "PR Number: $PR_NUMBER" + + # Fetch PR description from GitHub API (no auth needed for public repo) + API_RESPONSE=$(curl -s "https://api.github.com/repos/microsoft/mssql-python/pulls/$PR_NUMBER") + + # Debug: show API response status + echo "API Response (first 300 chars):" + echo "$API_RESPONSE" | head -c 300 + echo "" + + # Extract body using jq + PR_BODY=$(echo "$API_RESPONSE" | jq -r '.body // ""') + + echo "PR Body length: ${#PR_BODY}" + echo "PR Body preview:" + echo "$PR_BODY" | head -c 300 + echo "" + + # Extract mssql-tds PR number using sed (works on both Linux and macOS) + MSSQL_TDS_PR=$(echo "$PR_BODY" | sed -n 's/.*Depends-On:[[:space:]]*mssql-tds#\([0-9][0-9]*\).*/\1/p' | head -1) + + if [ -n "$MSSQL_TDS_PR" ]; then + echo "Found cross-repo dependency: mssql-tds#$MSSQL_TDS_PR" + echo "##vso[task.setvariable variable=MSSQL_TDS_PR_NUMBER]$MSSQL_TDS_PR" + echo "##vso[task.setvariable variable=HAS_CROSS_REPO_DEPENDENCY]true" + else + echo "No cross-repo dependency found in PR description" + echo "##vso[task.setvariable variable=HAS_CROSS_REPO_DEPENDENCY]false" + fi + else + echo "Not a PR build, skipping cross-repo dependency check" + echo "##vso[task.setvariable variable=HAS_CROSS_REPO_DEPENDENCY]false" + fi + displayName: 'Parse PR for mssql-tds dependency' + +# Download artifacts from mssql-tds pipeline if dependency found +- task: DownloadPipelineArtifact@2 + condition: eq(variables['HAS_CROSS_REPO_DEPENDENCY'], 'true') + displayName: 'Download mssql-py-core artifacts from mssql-tds' + inputs: + buildType: 'specific' + project: 'mssql-rs' + definition: '2204' # mssql-tds build and test pipeline + buildVersionToDownload: 'latestFromBranch' + branchName: 'refs/pull/$(MSSQL_TDS_PR_NUMBER)/merge' + artifactName: 'python-wheels-${{ parameters.osType }}-${{ parameters.architecture }}' + targetPath: '$(Pipeline.Workspace)/mssql-py-core-artifacts' + +# Extract and place .so files in mssql_python directory +- bash: | + echo "Processing mssql-py-core artifacts..." + + ARTIFACT_DIR="$(Pipeline.Workspace)/mssql-py-core-artifacts" + TARGET_DIR="$(Build.SourcesDirectory)/mssql_python" + + if [ -d "$ARTIFACT_DIR" ]; then + echo "Artifacts found in $ARTIFACT_DIR" + ls -la "$ARTIFACT_DIR" + + # Extract .so files from wheels or copy directly + for whl in "$ARTIFACT_DIR"/*.whl; do + if [ -f "$whl" ]; then + echo "Extracting wheel: $whl" + unzip -o "$whl" "*.so" -d "$TARGET_DIR" || true + unzip -o "$whl" "*.pyd" -d "$TARGET_DIR" || true + fi + done + + # Also copy any loose .so files + cp -f "$ARTIFACT_DIR"/*.so "$TARGET_DIR"/ 2>/dev/null || true + cp -f "$ARTIFACT_DIR"/*.pyd "$TARGET_DIR"/ 2>/dev/null || true + + echo "mssql_python directory contents:" + ls -la "$TARGET_DIR"/*.so 2>/dev/null || echo "No .so files found" + ls -la "$TARGET_DIR"/*.pyd 2>/dev/null || echo "No .pyd files found" + else + echo "No artifacts directory found - cross-repo dependency not configured" + fi + condition: eq(variables['HAS_CROSS_REPO_DEPENDENCY'], 'true') + displayName: 'Extract mssql-py-core .so files' diff --git a/scripts/build-py-core.sh b/scripts/build-py-core.sh new file mode 100755 index 00000000..246e238d --- /dev/null +++ b/scripts/build-py-core.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# Build mssql-py-core from mssql-tds repo and install into mssql-python +# +# Prerequisites: +# - Rust toolchain installed (https://rustup.rs/) +# - maturin installed: pip install maturin +# - Both mssql-python and mssql-tds repos cloned as siblings: +# parent-folder/ +# ├── mssql-python/ (this repo) +# └── mssql-tds/ (Rust TDS library) +# +# Usage: +# ./scripts/build-py-core.sh # Build and install .so to mssql_python/ +# ./scripts/build-py-core.sh --dev # Use maturin develop (faster, editable) +# ./scripts/build-py-core.sh --clean # Clean build artifacts first +# +# This script builds mssql_py_core (Rust-based Python bindings) and places +# the resulting .so file in the mssql_python/ directory alongside ddbc_bindings. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MSSQL_PYTHON_DIR="$(dirname "$SCRIPT_DIR")" +MSSQL_TDS_DIR="$(dirname "$MSSQL_PYTHON_DIR")/mssql-tds" +PY_CORE_DIR="$MSSQL_TDS_DIR/mssql-py-core" +TARGET_DIR="$MSSQL_PYTHON_DIR/mssql_python" + +# Parse arguments +USE_DEV_MODE=false +CLEAN_FIRST=false + +for arg in "$@"; do + case $arg in + --dev) + USE_DEV_MODE=true + ;; + --clean) + CLEAN_FIRST=true + ;; + --help|-h) + echo "Usage: $0 [--dev] [--clean]" + echo "" + echo "Options:" + echo " --dev Use 'maturin develop' for faster builds (requires venv)" + echo " --clean Clean build artifacts before building" + echo "" + exit 0 + ;; + esac +done + +echo "==========================================" +echo "Building mssql-py-core for mssql-python" +echo "==========================================" + +# Check if mssql-tds repo exists +if [ ! -d "$MSSQL_TDS_DIR" ]; then + echo "ERROR: mssql-tds repo not found at: $MSSQL_TDS_DIR" + echo "" + echo "Please clone mssql-tds as a sibling to mssql-python:" + echo " cd $(dirname "$MSSQL_PYTHON_DIR")" + echo " git clone mssql-tds" + exit 1 +fi + +if [ ! -d "$PY_CORE_DIR" ]; then + echo "ERROR: mssql-py-core directory not found at: $PY_CORE_DIR" + exit 1 +fi + +# Check for Rust toolchain +if ! command -v cargo &> /dev/null; then + echo "ERROR: Rust toolchain not found. Please install Rust:" + echo " curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" + exit 1 +fi + +# Check for maturin +if ! command -v maturin &> /dev/null; then + echo "ERROR: maturin not found. Please install:" + echo " pip install maturin" + exit 1 +fi + +# Clean if requested +if [ "$CLEAN_FIRST" = true ]; then + echo "[ACTION] Cleaning build artifacts..." + rm -rf "$PY_CORE_DIR/target" + rm -f "$TARGET_DIR"/mssql_py_core.*.so +fi + +# Get Python version info +PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')") +echo "[INFO] Python version: $PYTHON_VERSION" +echo "[INFO] mssql-tds path: $MSSQL_TDS_DIR" +echo "[INFO] Target directory: $TARGET_DIR" + +cd "$PY_CORE_DIR" + +if [ "$USE_DEV_MODE" = true ]; then + echo "" + echo "[ACTION] Running maturin develop (editable install)..." + maturin develop --release + + # Find and copy the .so file from the venv to mssql_python/ + echo "[ACTION] Copying .so to mssql_python/..." + + # Find the .so file in site-packages + SITE_PACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])") + SO_FILE=$(find "$SITE_PACKAGES" -name "mssql_py_core*.so" 2>/dev/null | head -1) + + if [ -n "$SO_FILE" ] && [ -f "$SO_FILE" ]; then + cp -f "$SO_FILE" "$TARGET_DIR/" + echo "[SUCCESS] Copied: $(basename "$SO_FILE")" + else + echo "[WARNING] .so file not found in site-packages, trying target directory..." + fi +else + echo "" + echo "[ACTION] Running maturin build --release..." + maturin build --release --out "$MSSQL_PYTHON_DIR/target/wheels" + + echo "" + echo "[ACTION] Extracting .so from wheel..." + + WHEEL_DIR="$MSSQL_PYTHON_DIR/target/wheels" + WHEEL_FILE=$(ls -1 "$WHEEL_DIR"/mssql_py_core-*.whl 2>/dev/null | head -1) + + if [ -n "$WHEEL_FILE" ] && [ -f "$WHEEL_FILE" ]; then + # Extract .so from wheel + unzip -o "$WHEEL_FILE" "*.so" -d "$TARGET_DIR/" 2>/dev/null || true + + # Move .so file if extracted to subdirectory + find "$TARGET_DIR" -name "mssql_py_core*.so" -exec mv {} "$TARGET_DIR/" \; 2>/dev/null || true + + echo "[SUCCESS] Wheel built: $(basename "$WHEEL_FILE")" + else + echo "[ERROR] No wheel file found in $WHEEL_DIR" + exit 1 + fi +fi + +# Verify the result +echo "" +echo "[RESULT] mssql_python directory contents:" +ls -la "$TARGET_DIR"/*.so 2>/dev/null || echo "No .so files found" + +# Check specifically for mssql_py_core +if ls "$TARGET_DIR"/mssql_py_core*.so 1>/dev/null 2>&1; then + echo "" + echo "==========================================" + echo "[SUCCESS] mssql-py-core built and installed!" + echo "==========================================" + echo "" + echo "You can now import mssql_py_core in Python:" + echo " from mssql_python import mssql_py_core" +else + echo "" + echo "[WARNING] mssql_py_core .so file not found in $TARGET_DIR" + echo "Check the build output above for errors." +fi