diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 0000000..7608d95 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,93 @@ +name: Build Artifacts + +permissions: + contents: read + +on: + push: + branches: + - main + paths-ignore: + - 'docs/**' + - 'licenses/**' + - '*.md' + - '.gitignore' + - '.gitattributes' + - 'LICENSE' + - 'NOTICE' + - '.github/**' + workflow_dispatch: + +jobs: + build-binaries: + name: Build Cross-Platform Binaries + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for proper version tagging + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Build all platforms + run: make build-all + env: + CGO_ENABLED: 0 + + - name: List built artifacts + run: | + echo "Built artifacts:" + ls -lh bin/ + + - name: Upload Windows AMD64 binary + uses: actions/upload-artifact@v4 + with: + name: flashpipex-windows-amd64 + path: bin/flashpipex-windows-amd64.exe + if-no-files-found: error + retention-days: 90 + + - name: Upload Linux AMD64 binary + uses: actions/upload-artifact@v4 + with: + name: flashpipex-linux-amd64 + path: bin/flashpipex-linux-amd64 + if-no-files-found: error + retention-days: 90 + + - name: Upload Linux ARM64 binary + uses: actions/upload-artifact@v4 + with: + name: flashpipex-linux-arm64 + path: bin/flashpipex-linux-arm64 + if-no-files-found: error + retention-days: 90 + + - name: Upload macOS AMD64 binary + uses: actions/upload-artifact@v4 + with: + name: flashpipex-darwin-amd64 + path: bin/flashpipex-darwin-amd64 + if-no-files-found: error + retention-days: 90 + + - name: Upload macOS ARM64 binary + uses: actions/upload-artifact@v4 + with: + name: flashpipex-darwin-arm64 + path: bin/flashpipex-darwin-arm64 + if-no-files-found: error + retention-days: 90 + + - name: Upload all binaries as single archive + uses: actions/upload-artifact@v4 + with: + name: flashpipex-all-platforms + path: bin/* + if-no-files-found: error + retention-days: 90 diff --git a/DOCUMENTATION_CONSOLIDATION.md b/DOCUMENTATION_CONSOLIDATION.md new file mode 100644 index 0000000..ee5c5a2 --- /dev/null +++ b/DOCUMENTATION_CONSOLIDATION.md @@ -0,0 +1,231 @@ +# Documentation Consolidation Summary + +**Date:** January 8, 2026 + +## Overview + +The FlashPipe documentation has been reorganized to clearly separate user-facing documentation from internal development documentation, and all example files have been consolidated. + +## Changes Made + +### 1. Created `dev-docs/` Directory + +Moved 8 internal development documentation files to `dev-docs/`: + +- ✅ `CLI_PORTING_SUMMARY.md` - CLI porting technical details +- ✅ `ORCHESTRATOR_ENHANCEMENTS.md` - Enhancement implementation details +- ✅ `PARTNER_DIRECTORY_MIGRATION.md` - Partner Directory technical migration +- ✅ `TESTING.md` - Testing guide for contributors +- ✅ `TEST_COVERAGE_SUMMARY.md` - Test coverage reports +- ✅ `TEST_QUICK_REFERENCE.md` - Testing quick reference +- ✅ `UNIT_TESTING_COMPLETION.md` - Test completion status +- ✅ `README.md` (new) - Index for dev documentation + +### 2. Moved User-Facing Documentation to `docs/` + +- ✅ `ORCHESTRATOR_MIGRATION.md` → `docs/orchestrator-migration.md` (migration guide for users) +- ✅ Removed duplicate `ORCHESTRATOR_QUICK_START.md` (already exists in docs/) + +### 3. Consolidated Example Files in `docs/examples/` + +Moved all example YAML files from root to `docs/examples/`: + +- ✅ `orchestrator-config-example.yml` +- ✅ `flashpipe-cpars-example.yml` +- ✅ `flashpipe-cpars.yml` +- ✅ Removed duplicate `orchestrator-config-example copy.yml` + +### 4. Created Missing Documentation + +- ✅ `docs/config-generate.md` - Comprehensive documentation for the `config-generate` command + +### 5. Updated README.md + +Enhanced the main README with: + +- ✅ Comprehensive "Enhanced Capabilities" section highlighting all new commands: + - 🎯 Orchestrator Command + - ⚙️ Config Generation + - 📁 Partner Directory Management +- ✅ Reorganized documentation section with clear categories: + - New Commands Documentation + - Migration Guides + - Core FlashPipe Documentation + - Examples + - Developer Documentation +- ✅ Updated all documentation links to reflect new file locations +- ✅ Added reference to `dev-docs/` for contributors + +## Final Directory Structure + +### Top-Level (Clean!) + +``` +ci-helper/ +├── README.md ← Main project README +├── CONTRIBUTING.md ← Contribution guidelines +├── CODE_OF_CONDUCT.md ← Code of conduct +├── LICENSE ← License file +├── NOTICE ← Notice file +├── docs/ ← User documentation +├── dev-docs/ ← Developer documentation (NEW) +├── internal/ ← Source code +├── cmd/ ← CLI entry point +└── ... +``` + +### docs/ (User Documentation) + +``` +docs/ +├── README files and guides +├── orchestrator.md ← Orchestrator comprehensive guide +├── orchestrator-quickstart.md ← Quick start guide +├── orchestrator-yaml-config.md ← YAML config reference +├── orchestrator-migration.md ← Migration from standalone CLI (MOVED) +├── config-generate.md ← Config generation guide (NEW) +├── partner-directory.md ← Partner Directory guide +├── partner-directory-config-examples.md +├── flashpipe-cli.md ← Core CLI reference +├── oauth_client.md ← OAuth setup +├── documentation.md ← General documentation +├── release-notes.md ← Release notes +└── examples/ ← Example configurations + ├── orchestrator-config-example.yml (MOVED) + ├── flashpipe-cpars-example.yml (MOVED) + ├── flashpipe-cpars.yml (MOVED) + └── flashpipe-config-with-orchestrator.yml +``` + +### dev-docs/ (Developer Documentation - NEW) + +``` +dev-docs/ +├── README.md ← Index (NEW) +├── CLI_PORTING_SUMMARY.md (MOVED) +├── ORCHESTRATOR_ENHANCEMENTS.md (MOVED) +├── PARTNER_DIRECTORY_MIGRATION.md (MOVED) +├── TESTING.md (MOVED) +├── TEST_COVERAGE_SUMMARY.md (MOVED) +├── TEST_QUICK_REFERENCE.md (MOVED) +└── UNIT_TESTING_COMPLETION.md (MOVED) +``` + +## Benefits + +### For Users + +1. **Cleaner Repository Root**: Only essential files (README, CONTRIBUTING, CODE_OF_CONDUCT, LICENSE) +2. **Clear Documentation Structure**: User docs in `docs/`, examples in `docs/examples/` +3. **Better Navigation**: README now has comprehensive sections linking to all features +4. **Complete Command Documentation**: All 4 new commands fully documented + +### For Contributors + +1. **Dedicated Dev Docs**: All development/internal docs in one place (`dev-docs/`) +2. **Clear Separation**: Easy to distinguish user-facing vs internal documentation +3. **Dev Docs Index**: `dev-docs/README.md` provides quick navigation + +### For Maintainability + +1. **No Duplicate Files**: Removed duplicate ORCHESTRATOR_QUICK_START.md and example files +2. **Logical Organization**: Related files grouped together +3. **Updated Cross-References**: All internal links updated to reflect new structure + +## Commands Documented + +All 4 new FlashPipe commands now have comprehensive documentation: + +1. **`flashpipe orchestrator`** - [docs/orchestrator.md](docs/orchestrator.md) + - Complete deployment lifecycle orchestration + - YAML configuration support + - Parallel deployment capabilities + - Environment prefix support + +2. **`flashpipe config-generate`** - [docs/config-generate.md](docs/config-generate.md) ⭐ NEW + - Automatic configuration generation + - Smart metadata extraction + - Config merging capabilities + - Filtering support + +3. **`flashpipe pd-snapshot`** - [docs/partner-directory.md](docs/partner-directory.md) + - Download Partner Directory parameters + - String and binary parameter support + - Batch operations + +4. **`flashpipe pd-deploy`** - [docs/partner-directory.md](docs/partner-directory.md) + - Upload Partner Directory parameters + - Full sync mode + - Dry run capability + +## Migration Impact + +### For Existing Users + +**No Breaking Changes!** All documentation has been moved but: +- Old links in external references may need updating +- All functionality remains the same +- Examples are now easier to find in `docs/examples/` + +### Recommended Updates + +If you have external documentation or scripts referencing old paths: + +```diff +- ORCHESTRATOR_MIGRATION.md ++ docs/orchestrator-migration.md + +- orchestrator-config-example.yml ++ docs/examples/orchestrator-config-example.yml + +- flashpipe-cpars-example.yml ++ docs/examples/flashpipe-cpars-example.yml +``` + +## Next Steps + +1. ✅ All files organized +2. ✅ README updated +3. ✅ Missing documentation created +4. ✅ Cross-references updated +5. 📝 Consider updating GitHub Pages site to reflect new structure +6. 📝 Update any CI/CD pipelines referencing old example paths + +## Verification + +Run these commands to verify the structure: + +```bash +# Top level should only have essential markdown +ls *.md +# Expected: README.md, CONTRIBUTING.md, CODE_OF_CONDUCT.md + +# Top level should have no example YAML files +ls *.yml +# Expected: (empty) + +# Dev docs should have 8 files +ls dev-docs/ +# Expected: 8 markdown files including README.md + +# Examples should have 4 YAML files +ls docs/examples/ +# Expected: 4 YAML files + +# Docs should include new config-generate.md +ls docs/config-generate.md +# Expected: Found +``` + +## Summary + +✅ **8 development documentation files** moved to `dev-docs/` +✅ **3 example YAML files** consolidated in `docs/examples/` +✅ **1 user migration guide** moved to `docs/` +✅ **1 new documentation file** created (`config-generate.md`) +✅ **1 dev-docs index** created +✅ **README.md** comprehensively updated with all new features +✅ **Top-level directory** cleaned up (only essential files remain) + +**Result:** Clear, organized, maintainable documentation structure! 🎉 + diff --git a/README.md b/README.md index ffe0730..e719fb4 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,90 @@ Integration (CI) & Continuous Delivery/Deployment (CD) capabilities for SAP Inte _FlashPipe_ aims to simplify the Build-To-Deploy cycle for SAP Integration Suite by providing CI/CD capabilities for automating time-consuming manual tasks. +### Enhanced Capabilities + +_FlashPipe_ has been significantly enhanced with powerful new commands for streamlined CI/CD workflows: + +#### 🎯 Orchestrator Command + +High-level deployment orchestration with integrated workflow management: + +- **Complete Lifecycle**: Update and deploy packages and artifacts in a single command +- **Multi-Source Configs**: Load from files, folders, or remote URLs +- **YAML Configuration**: Define all settings in a config file for reproducibility +- **Parallel Deployment**: Deploy multiple artifacts simultaneously (3-5x faster) +- **Environment Support**: Multi-tenant/environment prefixes (DEV, QA, PROD) +- **Selective Processing**: Filter by specific packages or artifacts + +```bash +# Simple deployment with YAML config +flashpipe orchestrator --orchestrator-config ./orchestrator.yml + +# Or with individual flags +flashpipe orchestrator --update \ + --deployment-prefix DEV \ + --deploy-config ./001-deploy-config.yml \ + --packages-dir ./packages +``` + +#### ⚙️ Config Generation + +Automatically generate deployment configurations from your packages directory: + +```bash +# Generate config from package structure +flashpipe config-generate --packages-dir ./packages --output ./deploy-config.yml +``` + +#### 📁 Partner Directory Management + +Snapshot and deploy Partner Directory parameters: + +```bash +# Download parameters from SAP CPI +flashpipe pd-snapshot --output ./partner-directory + +# Upload parameters to SAP CPI +flashpipe pd-deploy --source ./partner-directory +``` + +See documentation below for complete details on each command. + ### Documentation -For details on using _FlashPipe_, visit the [GitHub Pages documentation site](https://engswee.github.io/flashpipe/). +For comprehensive documentation on using _FlashPipe_, visit the [GitHub Pages documentation site](https://engswee.github.io/flashpipe/). + +#### New Commands Documentation + +- **[Orchestrator](docs/orchestrator.md)** - High-level deployment orchestration and workflow management +- **[Orchestrator Quick Start](docs/orchestrator-quickstart.md)** - Get started with orchestrator in 30 seconds +- **[Orchestrator YAML Config](docs/orchestrator-yaml-config.md)** - Complete YAML configuration reference +- **[Configure](docs/configure.md)** - Configure artifact parameters with YAML files +- **[Config Generate](docs/config-generate.md)** - Automatically generate deployment configurations +- **[Partner Directory](docs/partner-directory.md)** - Manage Partner Directory parameters + +#### Migration Guides + +- **[Orchestrator Migration Guide](docs/orchestrator-migration.md)** - Migrate from standalone CLI to integrated orchestrator + +#### Core FlashPipe Documentation + +- **[FlashPipe CLI Reference](docs/flashpipe-cli.md)** - Complete CLI command reference +- **[OAuth Client Setup](docs/oauth_client.md)** - Configure OAuth authentication +- **[GitHub Actions Integration](docs/documentation.md)** - CI/CD pipeline examples + +#### Examples + +Configuration examples are available in [docs/examples/](docs/examples/): +- `orchestrator-config-example.yml` - Orchestrator configuration template +- `flashpipe-cpars-example.yml` - Partner Directory configuration example + +#### Developer Documentation + +For contributors and maintainers, see [dev-docs/](dev-docs/) for: +- Testing guides and coverage reports +- CLI porting summaries +- Enhancement documentation ### Analytics diff --git a/docs/DOCUMENTATION_CLEANUP.md b/docs/DOCUMENTATION_CLEANUP.md new file mode 100644 index 0000000..323cdb8 --- /dev/null +++ b/docs/DOCUMENTATION_CLEANUP.md @@ -0,0 +1,129 @@ +# Documentation Cleanup Summary + +## Date +January 2024 + +## Overview +Consolidated and cleaned up repetitive Configure command documentation to reduce redundancy and improve maintainability. + +## Files Removed + +### Root Directory +- `CONFIGURE_COMMAND.md` - Removed (redundant) +- `CONFIGURE_FEATURE_README.md` - Removed (development artifact) +- `CONFIGURE_QUICK_REFERENCE.md` - Removed (redundant) +- `IMPLEMENTATION_SUMMARY.md` - Removed (development artifact) +- `YAML_CONFIG_IMPLEMENTATION.md` - Removed (development artifact) + +### docs/ Directory +- `docs/CONFIGURE_COMMAND_GUIDE.md` - Removed (1126 lines, too verbose) +- `docs/CONFIGURE_QUICK_REFERENCE.md` - Removed (redundant) + +**Total Removed:** 7 files + +## Files Created + +### docs/ Directory +- `docs/configure.md` - **New consolidated documentation** (418 lines) + - Clean, concise format + - Complete configuration reference + - Essential examples only + - Troubleshooting guide + - Best practices + +## What Was Consolidated + +The new `configure.md` combines: +1. Command overview and use cases +2. Configuration file format (complete reference) +3. Command-line flags +4. Global configuration options +5. 4 focused examples (instead of 9+) +6. Multi-environment deployment strategies +7. Troubleshooting guide +8. Best practices + +## Benefits + +✅ **Single Source of Truth:** One authoritative configure documentation file +✅ **Reduced Redundancy:** Eliminated duplicate content across 7 files +✅ **Easier Maintenance:** Update one file instead of many +✅ **Better UX:** Users find what they need quickly +✅ **Cleaner Repo:** Removed development artifacts from main branch + +## Documentation Structure (After Cleanup) + +``` +ci-helper/ +├── README.md # Main project README +├── configure-example.yml # Complete example config +├── config-examples/ # Multi-file examples +│ ├── README.md +│ ├── package1-database.yml +│ └── package2-api.yml +├── YAML_CONFIG.md # Global flashpipe.yaml reference +└── docs/ + ├── index.md # Documentation index (updated) + ├── configure.md # ⭐ NEW: Consolidated configure docs + ├── orchestrator.md # Orchestrator command + ├── config-generate.md # Config generation + ├── partner-directory.md # Partner Directory + ├── flashpipe-cli.md # CLI reference + └── oauth_client.md # Authentication setup +``` + +## Key Changes to Existing Files + +### README.md +- Added link to `docs/configure.md` + +### docs/index.md +- Updated to include Configure command +- Reorganized for better navigation + +## Example Reduction + +**Before:** 9+ lengthy examples scattered across multiple files +**After:** 4 focused examples in one file +- Example 1: Basic Configuration +- Example 2: Configure and Deploy +- Example 3: Folder-Based +- Example 4: Filtered Configuration + +Plus 3 multi-environment strategies (concise) + +## Recommendations + +1. **Keep Example Files:** `configure-example.yml` and `config-examples/` are still valuable +2. **Update Links:** If any external docs link to removed files, update them to `docs/configure.md` +3. **Version Control:** Tag this cleanup for future reference +4. **Future Additions:** Add new content to `docs/configure.md` only + +## Migration Path for Users + +If users bookmarked old documentation: + +| Old File | New Location | +|----------|--------------| +| `CONFIGURE_COMMAND.md` | `docs/configure.md` | +| `CONFIGURE_FEATURE_README.md` | `docs/configure.md` | +| `CONFIGURE_QUICK_REFERENCE.md` | `docs/configure.md` | +| `docs/CONFIGURE_COMMAND_GUIDE.md` | `docs/configure.md` | +| `docs/CONFIGURE_QUICK_REFERENCE.md` | `docs/configure.md` | + +## Next Steps + +1. ✅ Documentation consolidated +2. ✅ README updated +3. ✅ Index updated +4. 🔲 Test all documentation links +5. 🔲 Update any CI/CD pipelines referencing old docs +6. 🔲 Announce changes to users (if applicable) + +## Notes + +- All essential information preserved +- No functionality changes +- Examples simplified but remain complete +- Configuration reference fully intact +- Troubleshooting section enhanced \ No newline at end of file diff --git a/docs/config-generate.md b/docs/config-generate.md new file mode 100644 index 0000000..15c9ccf --- /dev/null +++ b/docs/config-generate.md @@ -0,0 +1,342 @@ +# Config Generate Command + +The `config-generate` command automatically generates or updates deployment configuration files by scanning your packages directory structure. + +## Overview + +The config generator scans your local packages directory and creates a deployment configuration file (`001-deploy-config.yml`) that can be used with the orchestrator command. It intelligently: + +- **Extracts metadata** from package JSON files and artifact MANIFEST.MF files +- **Preserves existing settings** when updating an existing configuration +- **Merges new discoveries** with your existing configuration +- **Filters** by specific packages or artifacts when needed + +## Usage + +```bash +flashpipe config-generate [flags] +``` + +### Basic Examples + +```bash +# Generate config with defaults (./packages → ./001-deploy-config.yml) +flashpipe config-generate + +# Specify custom directories +flashpipe config-generate \ + --packages-dir ./my-packages \ + --output ./my-config.yml + +# Generate config for specific packages only +flashpipe config-generate \ + --package-filter "DeviceManagement,OrderProcessing" + +# Generate config for specific artifacts only +flashpipe config-generate \ + --artifact-filter "OrderSync,DeviceSync" + +# Combine filters +flashpipe config-generate \ + --package-filter "DeviceManagement" \ + --artifact-filter "MDMDeviceSync,DeviceStatusUpdate" +``` + +## Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--packages-dir` | `./packages` | Path to packages directory to scan | +| `--output` | `./001-deploy-config.yml` | Path to output configuration file | +| `--package-filter` | (none) | Comma-separated list of package names to include | +| `--artifact-filter` | (none) | Comma-separated list of artifact names to include | + +## How It Works + +### 1. Directory Scanning + +The generator scans the packages directory with this expected structure: + +``` +packages/ +├── DeviceManagement/ +│ ├── DeviceManagement.json # Package metadata (optional) +│ ├── MDMDeviceSync/ +│ │ └── META-INF/MANIFEST.MF # Artifact metadata +│ └── DeviceStatusUpdate/ +│ └── META-INF/MANIFEST.MF +└── OrderProcessing/ + ├── OrderProcessing.json + └── OrderSync/ + └── META-INF/MANIFEST.MF +``` + +### 2. Metadata Extraction + +**From Package JSON** (e.g., `DeviceManagement.json`): +```json +{ + "Id": "DeviceManagement", + "Name": "Device Management Integration", + "Description": "Handles device synchronization", + "ShortText": "Device Sync" +} +``` + +**From MANIFEST.MF**: +``` +Manifest-Version: 1.0 +Bundle-SymbolicName: MDMDeviceSync +Bundle-Name: MDM Device Synchronization +SAP-BundleType: IntegrationFlow +``` + +Extracts: +- `Bundle-Name` → `displayName` +- `SAP-BundleType` → `type` (e.g., IntegrationFlow, MessageMapping, ScriptCollection) + +### 3. Smart Merging + +When updating an existing configuration: + +**Preserved:** +- ✅ `sync` and `deploy` flags +- ✅ `configOverrides` settings +- ✅ Custom display names and descriptions +- ✅ Deployment prefix + +**Added:** +- ✅ Newly discovered packages and artifacts +- ✅ Missing metadata fields + +**Removed:** +- ❌ Packages/artifacts no longer in directory (when not using filters) + +### 4. Generated Configuration + +Example output (`001-deploy-config.yml`): + +```yaml +deploymentPrefix: "" +packages: + - integrationSuiteId: DeviceManagement + packageDir: DeviceManagement + displayName: Device Management Integration + description: Handles device synchronization + short_text: Device Sync + sync: true + deploy: true + artifacts: + - artifactId: MDMDeviceSync + artifactDir: MDMDeviceSync + displayName: MDM Device Synchronization + type: IntegrationFlow + sync: true + deploy: true + configOverrides: {} + - artifactId: DeviceStatusUpdate + artifactDir: DeviceStatusUpdate + displayName: Device Status Update Flow + type: IntegrationFlow + sync: true + deploy: true + configOverrides: {} +``` + +## Filtering Behavior + +### Package Filter + +When using `--package-filter`: +- Only specified packages are processed +- Existing packages NOT in the filter are **preserved** in the output +- Statistics show filtered packages separately + +```bash +# Only process DeviceManagement, but keep others in config +flashpipe config-generate --package-filter "DeviceManagement" +``` + +### Artifact Filter + +When using `--artifact-filter`: +- Only specified artifacts are processed across all packages +- Existing artifacts NOT in the filter are **preserved** in the output +- Works across package boundaries + +```bash +# Only process specific artifacts regardless of package +flashpipe config-generate --artifact-filter "MDMDeviceSync,OrderSync" +``` + +### Combined Filters + +Both filters can be used together: + +```bash +# Only process MDMDeviceSync artifact in DeviceManagement package +flashpipe config-generate \ + --package-filter "DeviceManagement" \ + --artifact-filter "MDMDeviceSync" +``` + +## Statistics Report + +After generation, the command displays statistics: + +``` +Configuration generation completed successfully: + +Packages: + - Preserved: 2 + - Added: 1 + - Filtered: 1 + - Properties extracted: 1 + - Properties preserved: 2 + +Artifacts: + - Preserved: 8 + - Added: 2 + - Filtered: 3 + - Display names extracted: 2 + - Display names preserved: 8 + - Types extracted: 2 + - Types preserved: 8 + +Configuration written to: ./001-deploy-config.yml +``` + +## Use Cases + +### Initial Configuration + +Generate a complete configuration from scratch: + +```bash +# First time - creates new config +flashpipe config-generate +``` + +### Update After Changes + +After adding new packages or artifacts: + +```bash +# Updates existing config, adds new items +flashpipe config-generate +``` + +### Generate Subset Configuration + +Create configuration for a specific subset: + +```bash +# Generate config for QA-specific packages +flashpipe config-generate \ + --package-filter "QATestPackage1,QATestPackage2" \ + --output ./qa-deploy-config.yml +``` + +### Migration/Validation + +Regenerate to ensure consistency: + +```bash +# Regenerate to validate current structure +flashpipe config-generate --output ./validated-config.yml +``` + +## Best Practices + +1. **Commit Generated Configs**: Add generated files to version control +2. **Review Before Deploying**: Always review generated configs before deployment +3. **Use Filters for Large Projects**: Filter by package/artifact when working with specific components +4. **Preserve Custom Overrides**: The generator never removes your `configOverrides` settings +5. **Regular Updates**: Run after structural changes to your packages directory + +## Integration with Orchestrator + +The generated configuration is designed to work seamlessly with the orchestrator: + +```bash +# Generate configuration +flashpipe config-generate + +# Deploy using generated config +flashpipe orchestrator \ + --update \ + --deploy-config ./001-deploy-config.yml \ + --packages-dir ./packages \ + --tmn-host tenant.hana.ondemand.com \ + --oauth-host tenant.authentication.sap.hana.ondemand.com \ + --oauth-clientid your-client-id \ + --oauth-clientsecret your-client-secret +``` + +## Troubleshooting + +### Package Metadata Not Found + +If package JSON files don't exist, the generator will still create the configuration but with minimal metadata: + +```yaml +- integrationSuiteId: MyPackage + packageDir: "" + displayName: "" + description: "" + short_text: "" + sync: true + deploy: true +``` + +**Solution**: Create a `{PackageName}.json` file in the package directory. + +### Artifact Type Not Detected + +If MANIFEST.MF is missing or doesn't have `SAP-BundleType`: + +```yaml +- artifactId: MyArtifact + type: "" +``` + +**Solution**: Ensure MANIFEST.MF exists and contains `SAP-BundleType` header. + +### Existing Config Overwritten + +The generator preserves most settings but reorganizes the structure. + +**Solution**: Always review the diff before committing changes. Use version control. + +### Filter Not Working + +Filters are case-sensitive and must match exactly. + +**Solution**: Use exact package/artifact names as they appear in the directory structure. + +## Related Documentation + +- [Orchestrator Command](orchestrator.md) - Deploy using generated configurations +- [Orchestrator YAML Config](orchestrator-yaml-config.md) - Complete orchestrator configuration reference +- [Migration Guide](orchestrator-migration.md) - Migrating from standalone CLI + +## Example Workflow + +A typical workflow combining config generation and deployment: + +```bash +# 1. Sync from SAP CPI to local (if needed) +flashpipe snapshot --sync-package-details + +# 2. Generate deployment configuration +flashpipe config-generate + +# 3. Review generated configuration +cat ./001-deploy-config.yml + +# 4. Deploy using orchestrator +flashpipe orchestrator \ + --update \ + --deploy-config ./001-deploy-config.yml +``` + diff --git a/docs/configure.md b/docs/configure.md new file mode 100644 index 0000000..b46ccda --- /dev/null +++ b/docs/configure.md @@ -0,0 +1,418 @@ +# Configure Command + +Configure SAP Cloud Integration artifact parameters using declarative YAML files. + +## Table of Contents + +- [Overview](#overview) +- [Quick Start](#quick-start) +- [Configuration File Format](#configuration-file-format) +- [Command Reference](#command-reference) +- [Examples](#examples) +- [Multi-Environment Deployments](#multi-environment-deployments) +- [Troubleshooting](#troubleshooting) + +--- + +## Overview + +The `configure` command updates configuration parameters for SAP CPI artifacts and optionally deploys them. + +**Key Features:** +- Declarative YAML-based configuration +- Batch operations for efficient parameter updates +- Optional deployment after configuration +- Multi-environment support via deployment prefixes +- Dry-run mode to preview changes +- Process single file or folder of configs + +**Use Cases:** +- Environment promotion (DEV → QA → PROD) +- Bulk parameter updates +- Configuration as code in CI/CD pipelines +- Disaster recovery + +--- + +## Quick Start + +**1. Create config file (`my-config.yml`):** + +```yaml +packages: + - integrationSuiteId: "MyPackage" + displayName: "My Integration Package" + + artifacts: + - artifactId: "MyFlow" + displayName: "My Integration Flow" + type: "Integration" + version: "active" + deploy: true + + parameters: + - key: "DatabaseURL" + value: "jdbc:mysql://localhost:3306/mydb" + - key: "APIKey" + value: "${env:API_KEY}" +``` + +**2. Set environment variables:** + +```bash +export API_KEY="your-secret-key" +``` + +**3. Run command:** + +```bash +# Preview changes +flashpipe configure --config-path ./my-config.yml --dry-run + +# Apply configuration +flashpipe configure --config-path ./my-config.yml +``` + +--- + +## Configuration File Format + +### Complete Structure + +```yaml +# Optional: Deployment prefix for all packages/artifacts +deploymentPrefix: "DEV_" + +packages: + - integrationSuiteId: "PackageID" # Required + displayName: "Package Display Name" # Required + deploy: false # Optional: deploy all artifacts in package + + artifacts: + - artifactId: "ArtifactID" # Required + displayName: "Artifact Name" # Required + type: "Integration" # Required: Integration|MessageMapping|ScriptCollection|ValueMapping + version: "active" # Optional: default "active" + deploy: true # Optional: deploy this artifact after config + + parameters: + - key: "ParameterName" # Required + value: "ParameterValue" # Required + + batch: # Optional batch settings + enabled: true # default: true + batchSize: 90 # default: 90 +``` + +### Field Reference + +#### Package + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `integrationSuiteId` | string | Yes | Package ID in SAP CPI | +| `displayName` | string | Yes | Package display name | +| `deploy` | boolean | No | Deploy all artifacts in package (default: false) | +| `artifacts` | array | Yes | List of artifacts to configure | + +#### Artifact + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `artifactId` | string | Yes | Artifact ID in SAP CPI | +| `displayName` | string | Yes | Artifact display name | +| `type` | string | Yes | `Integration`, `MessageMapping`, `ScriptCollection`, or `ValueMapping` | +| `version` | string | No | Version to configure (default: "active") | +| `deploy` | boolean | No | Deploy after configuration (default: false) | +| `parameters` | array | Yes | Configuration parameters | +| `batch` | object | No | Batch processing settings | + +#### Parameter + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `key` | string | Yes | Parameter name | +| `value` | string | Yes | Parameter value (supports `${env:VAR}` syntax) | + +### Environment Variables + +Reference environment variables using `${env:VARIABLE_NAME}`: + +```yaml +parameters: + - key: "DatabasePassword" + value: "${env:DB_PASSWORD}" + - key: "OAuthSecret" + value: "${env:OAUTH_SECRET}" +``` + +--- + +## Command Reference + +### Syntax + +```bash +flashpipe configure [flags] +``` + +### Flags + +| Flag | Short | Type | Default | Description | +|------|-------|------|---------|-------------| +| `--config-path` | `-c` | string | *required* | Path to YAML file or folder | +| `--deployment-prefix` | `-p` | string | `""` | Prefix for package/artifact IDs | +| `--package-filter` | | string | `""` | Filter packages (comma-separated) | +| `--artifact-filter` | | string | `""` | Filter artifacts (comma-separated) | +| `--dry-run` | | bool | `false` | Preview without applying | +| `--deploy-retries` | | int | `5` | Deployment status check retries | +| `--deploy-delay` | | int | `15` | Seconds between deployment checks | +| `--parallel-deployments` | | int | `3` | Max parallel deployments | +| `--batch-size` | | int | `90` | Parameters per batch request | +| `--disable-batch` | | bool | `false` | Disable batch processing | + +### Global Configuration (flashpipe.yaml) + +```yaml +configure: + configPath: "./config/dev" + deploymentPrefix: "DEV_" + dryRun: false + deployRetries: 5 + deployDelaySeconds: 15 + parallelDeployments: 3 + batchSize: 90 + disableBatch: false +``` + +Run without flags: +```bash +flashpipe configure +``` + +*Note: CLI flags override flashpipe.yaml settings.* + +--- + +## Examples + +### Example 1: Basic Configuration + +Update parameters without deployment: + +```yaml +packages: + - integrationSuiteId: "CustomerSync" + displayName: "Customer Synchronization" + + artifacts: + - artifactId: "CustomerDataFlow" + displayName: "Customer Data Integration" + type: "Integration" + deploy: false + + parameters: + - key: "SourceURL" + value: "https://erp.example.com/api/customers" + - key: "BatchSize" + value: "100" +``` + +```bash +flashpipe configure --config-path ./config.yml +``` + +### Example 2: Configure and Deploy + +Update parameters and deploy: + +```yaml +packages: + - integrationSuiteId: "OrderProcessing" + displayName: "Order Processing" + deploy: true + + artifacts: + - artifactId: "OrderValidation" + type: "Integration" + deploy: true + + parameters: + - key: "ValidationRules" + value: "STRICT" +``` + +```bash +flashpipe configure --config-path ./config.yml +``` + +### Example 3: Folder-Based + +Process all YAML files in a folder: + +``` +configs/ +├── package1.yml +├── package2.yml +└── package3.yml +``` + +```bash +flashpipe configure --config-path ./configs +``` + +### Example 4: Filtered Configuration + +Configure specific packages or artifacts: + +```bash +# Specific packages +flashpipe configure --config-path ./config.yml \ + --package-filter "Package1,Package2" + +# Specific artifacts +flashpipe configure --config-path ./config.yml \ + --artifact-filter "Flow1,Flow2" +``` + +--- + +## Multi-Environment Deployments + +### Strategy 1: Deployment Prefixes + +Use same config, different prefixes: + +```bash +# Development +flashpipe configure --config-path ./config.yml --deployment-prefix "DEV_" + +# QA +flashpipe configure --config-path ./config.yml --deployment-prefix "QA_" + +# Production +flashpipe configure --config-path ./config.yml --deployment-prefix "PROD_" +``` + +### Strategy 2: Separate Folders + +Environment-specific configs: + +``` +config/ +├── dev/ +│ └── flows.yml +├── qa/ +│ └── flows.yml +└── prod/ + └── flows.yml +``` + +```bash +flashpipe configure --config-path ./config/dev +flashpipe configure --config-path ./config/qa +flashpipe configure --config-path ./config/prod +``` + +### Strategy 3: Environment Variables + +```yaml +parameters: + - key: "ServiceURL" + value: "${env:SERVICE_URL}" + - key: "APIKey" + value: "${env:API_KEY}" +``` + +```bash +# Development +export SERVICE_URL="https://dev-api.example.com" +export API_KEY="dev-key" +flashpipe configure --config-path ./config.yml + +# Production +export SERVICE_URL="https://api.example.com" +export API_KEY="prod-key" +flashpipe configure --config-path ./config.yml +``` + +--- + +## Troubleshooting + +### Enable Debug Logging + +```bash +export FLASHPIPE_DEBUG=true +flashpipe configure --config-path ./config.yml +``` + +### Always Use Dry Run First + +```bash +flashpipe configure --config-path ./config.yml --dry-run +``` + +### Common Issues + +| Issue | Solution | +|-------|----------| +| Config file not found | Verify path, use absolute path | +| Invalid YAML syntax | Check indentation (spaces not tabs), validate online | +| Authentication failed | Verify credentials in `flashpipe.yaml` | +| Artifact not found | Check ID is correct (case-sensitive), verify prefix | +| Parameter update failed | Try `--disable-batch` flag | +| Deployment timeout | Increase `--deploy-retries` and `--deploy-delay` | +| Environment variable not substituted | Ensure `export` executed before command | + +### Summary Output + +The command prints detailed statistics: + +``` +═══════════════════════════════════════════════════════════════════════ +CONFIGURATION SUMMARY +═══════════════════════════════════════════════════════════════════════ + +Configuration Phase: + Packages processed: 2 + Artifacts processed: 5 + Artifacts configured: 5 + Parameters updated: 23 + +Processing Method: + Batch requests executed: 3 + Individual requests used: 0 + +Deployment Phase: + Deployments successful: 2 + Deployments failed: 0 + +Overall Status: ✅ SUCCESS +``` + +--- + +## Best Practices + +✅ **DO:** +- Use `--dry-run` before applying changes +- Version control configuration files +- Use environment variables for secrets +- Test in DEV before promoting to PROD +- Document parameters with comments + +❌ **DON'T:** +- Commit secrets to Git +- Skip dry-run in production +- Use hardcoded credentials +- Deploy without testing first + +--- + +## See Also + +- [configure-example.yml](../configure-example.yml) - Complete example +- [config-examples/](../config-examples/) - Multi-file examples +- [Orchestrator Command](orchestrator.md) - For full artifact deployments +- [OAuth Setup](oauth_client.md) - Authentication configuration \ No newline at end of file diff --git a/docs/examples/flashpipe-config-with-orchestrator.yml b/docs/examples/flashpipe-config-with-orchestrator.yml new file mode 100644 index 0000000..d98af39 --- /dev/null +++ b/docs/examples/flashpipe-config-with-orchestrator.yml @@ -0,0 +1,196 @@ +# Flashpipe Configuration File with Orchestrator Settings +# This file demonstrates how to configure the orchestrator using the global config file + +# SAP CPI Connection Settings (used by all commands) +host: https://your-tenant.integrationsuite.cfapps.eu10.hana.ondemand.com +username: your-username +password: your-password + +# OAuth settings (alternative to username/password) +# oauth-clientid: your-client-id +# oauth-clientsecret: your-client-secret +# oauth-host: your-tenant.authentication.eu10.hana.ondemand.com +# oauth-path: /oauth/token + +# Orchestrator Configuration +# All settings in this section can be overridden via CLI flags +orchestrator: + # Required Settings + packagesDir: ./packages # Directory containing your packages + deployConfig: ./deploy-config.yml # Path to deployment configuration + + # Optional: Filtering & Prefixing + deploymentPrefix: "" # Prefix for package/artifact IDs (e.g., "DEV", "PROD") + packageFilter: "" # Comma-separated package names to deploy + artifactFilter: "" # Comma-separated artifact names to deploy + + # Optional: Config Loading + configPattern: "*.y*ml" # File pattern when deployConfig is a folder + mergeConfigs: false # Merge multiple config files into one deployment + + # Optional: Execution + keepTemp: false # Keep temporary files for debugging + mode: "update-and-deploy" # Options: "update-and-deploy", "update-only", "deploy-only" + + # Optional: Parallel Deployment Settings + deployRetries: 5 # Number of status check retries per deployment + deployDelaySeconds: 15 # Seconds to wait between status checks + parallelDeployments: 3 # Max concurrent deployments per package + +# Partner Directory Configuration (optional) +pd-snapshot: + output: ./partner-directories + pids: [] + replace: false + +pd-deploy: + input: ./partner-directories + pids: [] + mode: "replace" + dry-run: false + full-sync: false + +--- +# Example: Development Environment Configuration +host: https://dev-tenant.integrationsuite.cfapps.eu10.hana.ondemand.com +username: dev-user +password: dev-password + +orchestrator: + packagesDir: ./packages + deployConfig: ./configs/dev + deploymentPrefix: DEV + mode: update-and-deploy + + # Fast deployment for development + parallelDeployments: 5 + deployRetries: 5 + deployDelaySeconds: 15 + + # Merge all config files in folder + mergeConfigs: true + configPattern: "*.yml" + +--- +# Example: Production Environment Configuration +host: https://prod-tenant.integrationsuite.cfapps.eu10.hana.ondemand.com +oauth-clientid: prod-client-id +oauth-clientsecret: prod-client-secret +oauth-host: prod-tenant.authentication.eu10.hana.ondemand.com + +orchestrator: + packagesDir: ./packages + deployConfig: ./configs/production.yml + deploymentPrefix: PROD + mode: update-and-deploy + + # Conservative settings for production + parallelDeployments: 2 # Lower parallelism for safety + deployRetries: 10 # More retries for reliability + deployDelaySeconds: 30 # Longer delays between checks + + mergeConfigs: false + keepTemp: false + +--- +# Example: CI/CD Pipeline Configuration +host: https://ci-tenant.integrationsuite.cfapps.eu10.hana.ondemand.com +username: ${CPI_USERNAME} +password: ${CPI_PASSWORD} + +orchestrator: + packagesDir: ./packages + # Load config from remote repository + deployConfig: https://raw.githubusercontent.com/myorg/configs/main/ci-config.yml + deploymentPrefix: CI + mode: update-and-deploy + + # Optimize for speed in CI/CD + parallelDeployments: 10 + deployRetries: 5 + deployDelaySeconds: 10 + +--- +# Example: Debugging/Testing Configuration +host: https://dev-tenant.integrationsuite.cfapps.eu10.hana.ondemand.com +username: dev-user +password: dev-password + +orchestrator: + packagesDir: ./packages + deployConfig: ./test-config.yml + deploymentPrefix: TEST + + # Focus on single package for testing + packageFilter: "TestPackage" + + # Debug settings + mode: update-only # Don't deploy, just update + keepTemp: true # Keep temp files for inspection + parallelDeployments: 1 # Single-threaded for easier debugging + +--- +# Example: Selective Deployment Configuration +host: https://qa-tenant.integrationsuite.cfapps.eu10.hana.ondemand.com +username: qa-user +password: qa-password + +orchestrator: + packagesDir: ./packages + deployConfig: ./configs/qa + deploymentPrefix: QA + + # Deploy only specific packages + packageFilter: "CustomerIntegration,DeviceManagement" + + # Deploy only specific artifacts within those packages + artifactFilter: "CustomerSync,DeviceStatusUpdate" + + mode: update-and-deploy + parallelDeployments: 3 + mergeConfigs: true + +--- +# Example: Multiple Configs from Folder +host: https://dev-tenant.integrationsuite.cfapps.eu10.hana.ondemand.com +username: dev-user +password: dev-password + +orchestrator: + packagesDir: ./packages + # Point to a folder containing multiple config files + deployConfig: ./configs/all-environments + deploymentPrefix: DEV + + # Process all .yml files in the folder + configPattern: "*.yml" + + # Merge all configs into a single deployment + # Each config can have its own prefix which will be applied + mergeConfigs: true + + mode: update-and-deploy + parallelDeployments: 5 + +--- +# Usage Examples: +# +# 1. Use config file with all defaults: +# flashpipe orchestrator --config ./flashpipe-dev.yml --update +# +# 2. Override specific settings via CLI: +# flashpipe orchestrator --config ./flashpipe-dev.yml \ +# --deployment-prefix OVERRIDE --parallel-deployments 10 +# +# 3. Use different config for different environments: +# flashpipe orchestrator --config ./flashpipe-dev.yml --update +# flashpipe orchestrator --config ./flashpipe-qa.yml --update +# flashpipe orchestrator --config ./flashpipe-prod.yml --update +# +# 4. Debug mode with temp files: +# flashpipe orchestrator --config ./flashpipe-test.yml \ +# --keep-temp --debug --update-only +# +# 5. Deploy specific packages only: +# flashpipe orchestrator --config ./flashpipe.yml \ +# --package-filter "Package1,Package2" diff --git a/docs/examples/flashpipe-cpars-example.yml b/docs/examples/flashpipe-cpars-example.yml new file mode 100644 index 0000000..d4d905f --- /dev/null +++ b/docs/examples/flashpipe-cpars-example.yml @@ -0,0 +1,109 @@ +# Example Flashpipe Configuration for Partner Directory Management +# Copy this file and customize for your environment + +# ============================================================================= +# Connection Settings (Required) +# ============================================================================= +# Use environment variables for sensitive credentials +tmn-host: your-tenant.hana.ondemand.com +oauth-host: your-tenant.authentication.sap.hana.ondemand.com +oauth-clientid: ${OAUTH_CLIENT_ID} +oauth-clientsecret: ${OAUTH_CLIENT_SECRET} + +# OR use Basic Authentication +# tmn-host: your-tenant.hana.ondemand.com +# tmn-userid: ${USERNAME} +# tmn-password: ${PASSWORD} + +# ============================================================================= +# Partner Directory Snapshot Settings +# ============================================================================= +pd-snapshot: + # Directory where parameters will be saved + resources-path: ./partner-directory + + # Replace existing files when downloading + # true = overwrite existing files + # false = only add new files, keep existing + replace: true + + # Optional: Only snapshot specific Partner IDs + # Uncomment and modify to filter + # pids: + # - SAP_SYSTEM_001 + # - CUSTOMER_API + # - PARTNER_XYZ + +# ============================================================================= +# Partner Directory Deploy Settings +# ============================================================================= +pd-deploy: + # Directory where parameters are read from + resources-path: ./partner-directory + + # Replace existing values in CPI + # true = update existing parameters with local values + # false = only create new parameters, skip existing + replace: true + + # Full synchronization mode (CAUTION!) + # true = delete remote parameters not present in local files + # false = only create/update, never delete + # WARNING: full-sync makes local files the source of truth! + full-sync: false + + # Dry-run mode - preview changes without applying them + # true = show what would change but don't modify CPI + # false = actually apply changes to CPI + dry-run: false + + # Optional: Only deploy specific Partner IDs + # Uncomment and modify to filter + # pids: + # - SAP_SYSTEM_001 + # - CUSTOMER_API + # - PARTNER_XYZ + +# ============================================================================= +# Usage Examples +# ============================================================================= +# +# Set credentials via environment variables: +# export OAUTH_CLIENT_ID="your-client-id" +# export OAUTH_CLIENT_SECRET="your-client-secret" +# +# Snapshot (download) Partner Directory: +# flashpipe pd-snapshot --config flashpipe-cpars-example.yml +# +# Deploy with dry-run (preview changes): +# flashpipe pd-deploy --config flashpipe-cpars-example.yml --dry-run +# +# Deploy (apply changes): +# flashpipe pd-deploy --config flashpipe-cpars-example.yml +# +# Deploy with full-sync (delete remote params not in local): +# flashpipe pd-deploy --config flashpipe-cpars-example.yml --full-sync +# +# Override resources path: +# flashpipe pd-deploy --config flashpipe-cpars-example.yml \ +# --resources-path ./other-directory +# +# Deploy only specific PIDs: +# flashpipe pd-deploy --config flashpipe-cpars-example.yml \ +# --pids "PARTNER_001,PARTNER_002" +# +# ============================================================================= +# Environment-Specific Configs +# ============================================================================= +# For multiple environments, create separate config files: +# - flashpipe-cpars-dev.yml +# - flashpipe-cpars-qa.yml +# - flashpipe-cpars-prod.yml +# +# Each with different: +# - tmn-host +# - oauth-host +# - resources-path +# - full-sync settings +# +# ============================================================================= diff --git a/docs/examples/orchestrator-config-example.yml b/docs/examples/orchestrator-config-example.yml new file mode 100644 index 0000000..492497f --- /dev/null +++ b/docs/examples/orchestrator-config-example.yml @@ -0,0 +1,70 @@ +# Orchestrator Configuration Example +# This file demonstrates all available orchestrator settings that can be configured via YAML + +# Directory containing the packages to deploy +packagesDir: ../../integration-toolkit/packages + +# Path to deployment configuration (file, folder, or URL) +# - Single file: ./001-deploy-config.yml +# - Folder: ./configs (processes all *.yml files) +# - Remote URL: https://raw.githubusercontent.com/org/repo/main/config.yml +deployConfig: ../../integration-toolkit/deployments + +# Optional: Deployment prefix for package/artifact IDs +# This will be prepended to all package and artifact IDs +# Example: "DEV" -> package becomes "DEVMyPackage" +deploymentPrefix: "" + +# Optional: Filter packages by name (comma-separated) +# Only these packages will be processed +# Example: "DeviceManagement,GenericPipeline" +packageFilter: "" + +# Optional: Filter artifacts by name (comma-separated) +# Only these artifacts will be processed +# Example: "MDMEquipmentMutationOutbound,CustomerDataSync" +artifactFilter: "" + +# Optional: File pattern for config files when using folder source +# Default: "*.y*ml" (matches both .yml and .yaml) +configPattern: "*.y*ml" + +# Optional: Merge multiple config files into single deployment +# Default: false (process each config separately) +# When true: All configs are merged and deployed together with their prefixes +mergeConfigs: true + +# Optional: Keep temporary directory after execution +# Default: false (cleanup temp files) +# Useful for debugging - temp dir contains modified artifacts +keepTemp: false + +# Operation mode: "update-and-deploy", "update-only", or "deploy-only" +# - update-and-deploy: Update artifacts, then deploy them (default) +# - update-only: Only update artifacts, skip deployment +# - deploy-only: Only deploy artifacts, skip updates +mode: "update-and-deploy" + +# Deployment Settings (Phase 2 - Parallel Deployment) + +# Number of status check retries for each deployment +# Default: 5 +# Each retry waits for deployDelaySeconds before checking again +deployRetries: 5 + +# Delay in seconds between deployment status checks +# Default: 15 +# Increase this if deployments take longer in your environment +deployDelaySeconds: 15 + +# Maximum number of parallel deployments per package +# Default: 3 +# Increase for faster deployments (but watch for rate limits) +# Decrease if you hit API rate limits or memory constraints +parallelDeployments: 10 + +tmn-host: cpars-it-dev-34m09zg5.it-cpi033.cfapps.eu10-005.hana.ondemand.com +oauth-host: cpars-it-dev-34m09zg5.authentication.eu10.hana.ondemand.com +oauth-clientid: sb-364df3c5-96d2-46a5-aa3d-dc10008a61c4!b298933|it!b543091 +oauth-clientsecret: 51846b81-b497-419c-be80-982d5d9dc870$GdmjuZEBPtot-pNUJU2CAchydiVeqbVPgH4g9mKENRY= +dir-work: $HOME/work \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 648e451..95b8254 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,20 +1,28 @@ FlashPipe Logo -# About FlashPipe +# FlashPipe Documentation _FlashPipe_ is a public [Docker image](https://hub.docker.com/r/engswee/flashpipe) that provides Continuous Integration (CI) & Continuous Delivery/Deployment (CD) capabilities for SAP Integration Suite. -_FlashPipe_ aims to simplify the Build-To-Deploy cycle for SAP Integration Suite by providing CI/CD capabilities for -automating time-consuming manual tasks. +## Command Documentation + +- **[Orchestrator](orchestrator.md)** - High-level deployment orchestration +- **[Configure](configure.md)** - Configure artifact parameters with YAML +- **[Config Generate](config-generate.md)** - Auto-generate deployment configs +- **[Partner Directory](partner-directory.md)** - Manage Partner Directory parameters +- **[FlashPipe CLI](flashpipe-cli.md)** - Complete CLI reference ## Getting Started -For details on how to start using _FlashPipe_, visit the [documentation page](documentation.md). +- **[Orchestrator Quick Start](orchestrator-quickstart.md)** - Get started in 30 seconds +- **[OAuth Client Setup](oauth_client.md)** - Configure authentication +- **[GitHub Actions Integration](documentation.md)** - CI/CD pipeline examples -## Release Notes +## Additional Resources -The version history and details of each release can be found in the [release notes](release-notes.md). +- **[Release Notes](release-notes.md)** - Version history +- **[Examples](examples/)** - Configuration examples ## License diff --git a/docs/orchestrator-migration.md b/docs/orchestrator-migration.md new file mode 100644 index 0000000..ae59a33 --- /dev/null +++ b/docs/orchestrator-migration.md @@ -0,0 +1,532 @@ +# Migration Guide: Standalone CLI to Integrated Orchestrator + +This guide helps you migrate from the standalone `ci-helper` CLI to the integrated Flashpipe orchestrator command. + +## Overview + +The standalone CLI has been **fully integrated** into Flashpipe as the `orchestrator` command. All functionality has been ported to use internal Flashpipe functions instead of spawning external processes. + +### What Changed + +**Before (Standalone CLI):** +- Separate `ci-helper` binary +- Called `flashpipe` binary as external process +- Required both binaries to be installed +- Multiple process spawns for each operation + +**After (Integrated Orchestrator):** +- Single `flashpipe` binary +- Uses internal Flashpipe functions directly +- Single process for entire deployment +- Better performance and error handling + +## Command Mapping + +### Flashpipe Wrapper Command + +**Old:** +```bash +ci-helper flashpipe --update \ + --packages-dir ./packages \ + --flashpipe-config ./flashpipe.yml \ + --deploy-config ./001-deploy-config.yml \ + --deployment-prefix DEV +``` + +**New (Recommended - using --config flag):** +```bash +flashpipe orchestrator --update \ + --config $HOME/flashpipe.yaml \ + --packages-dir ./packages \ + --deploy-config ./001-deploy-config.yml \ + --deployment-prefix DEV +``` + +**Alternative (using individual flags):** +```bash +flashpipe orchestrator --update \ + --packages-dir ./packages \ + --deploy-config ./001-deploy-config.yml \ + --deployment-prefix DEV \ + --tmn-host tenant.hana.ondemand.com \ + --oauth-host tenant.authentication.sap.hana.ondemand.com \ + --oauth-clientid your-client-id \ + --oauth-clientsecret your-client-secret +``` + +**Key Changes:** +- Command: `ci-helper flashpipe` → `flashpipe orchestrator` +- Connection details: `--flashpipe-config` → `--config` (standard Flashpipe flag) or individual flags +- The `--config` flag works exactly like other Flashpipe commands +- All other flags remain the same + +### Config Generate Command + +**Old:** +```bash +ci-helper config --packages-dir ./packages --output ./deploy-config.yml +``` + +**New:** +```bash +flashpipe config-generate --packages-dir ./packages --output ./deploy-config.yml +``` + +**Changes:** +- Command: `ci-helper config` → `flashpipe config-generate` +- All flags remain the same + +### Partner Directory Commands + +**Old:** +```bash +ci-helper pd snapshot --config ./pd-config.yml --output ./partner-directory +ci-helper pd deploy --config ./pd-config.yml --source ./partner-directory +``` + +**New:** +```bash +flashpipe pd-snapshot --config ./pd-config.yml --output ./partner-directory +flashpipe pd-deploy --config ./pd-config.yml --source ./partner-directory +``` + +**Changes:** +- Commands: `ci-helper pd snapshot` → `flashpipe pd-snapshot` +- Commands: `ci-helper pd deploy` → `flashpipe pd-deploy` +- All flags remain the same + +## Configuration Files + +### Deployment Config (No Changes) + +The deployment configuration format is **identical**: + +```yaml +deploymentPrefix: "DEV" +packages: + - integrationSuiteId: "DeviceManagement" + packageDir: "DeviceManagement" + displayName: "Device Management" + sync: true + deploy: true + artifacts: + - artifactId: "MDMDeviceSync" + artifactDir: "MDMDeviceSync" + type: "IntegrationFlow" + sync: true + deploy: true + configOverrides: + Timeout: "60000" +``` + +### Connection Config + +The orchestrator uses the **standard Flashpipe config file** format, just like all other Flashpipe commands. + +**Old (`flashpipe-config.yml` - standalone CLI format):** +```yaml +host: tenant.hana.ondemand.com +oauth: + host: tenant.authentication.sap.hana.ondemand.com + clientid: your-client-id + clientsecret: your-client-secret +``` + +**New (`$HOME/flashpipe.yaml` - standard Flashpipe format):** +```yaml +tmn-host: tenant.hana.ondemand.com +oauth-host: tenant.authentication.sap.hana.ondemand.com +oauth-clientid: your-client-id +oauth-clientsecret: your-client-secret +``` + +**Usage:** +```bash +# Auto-detected from $HOME/flashpipe.yaml +flashpipe orchestrator --update --deploy-config ./deploy-config.yml + +# Or specify custom location +flashpipe orchestrator --update \ + --config ./my-flashpipe.yaml \ + --deploy-config ./deploy-config.yml + +# Or use individual flags +flashpipe orchestrator --update \ + --tmn-host tenant.hana.ondemand.com \ + --oauth-host tenant.authentication.sap.hana.ondemand.com \ + --oauth-clientid your-client-id \ + --oauth-clientsecret your-client-secret \ + --deploy-config ./deploy-config.yml +``` + +**Recommendation:** Use the `--config` flag or place the config file at `$HOME/flashpipe.yaml` for automatic detection. This is more secure than passing credentials via command-line flags. + +## Flag Changes + +### Removed Flags + +These flags from the standalone CLI are **no longer needed**: + +- `--flashpipe-config` - Use `--config` or individual connection flags +- `--tmn-host`, `--oauth-*` (when using individual flags) - Now use standard Flashpipe flags + +### New Flags + +These flags are now available: + +- `--config` - Path to Flashpipe config file (standard across all commands) +- All standard Flashpipe connection flags + +### Renamed Flags + +| Old Flag | New Flag | Notes | +|----------|----------|-------| +| None | - | All flags kept the same name | + +## Directory Structure + +**No changes required** - the directory structure is identical: + +``` +. +├── packages/ +│ ├── DeviceManagement/ +│ │ ├── MDMDeviceSync/ +│ │ │ ├── META-INF/MANIFEST.MF +│ │ │ └── src/main/resources/parameters.prop +│ │ └── ... +│ └── ... +├── 001-deploy-config.yml +└── flashpipe.yaml (optional - for connection details) +``` + +## Step-by-Step Migration + +### Step 1: Install Latest Flashpipe + +Download the latest Flashpipe release (with orchestrator): + +```bash +# Linux/macOS +wget https://github.com/engswee/flashpipe/releases/latest/download/flashpipe-linux-amd64 +chmod +x flashpipe-linux-amd64 +sudo mv flashpipe-linux-amd64 /usr/local/bin/flashpipe + +# Windows +# Download flashpipe-windows-amd64.exe from releases +# Rename to flashpipe.exe +# Add to PATH +``` + +### Step 2: Update Scripts/CI Pipelines + +Replace `ci-helper` commands with `flashpipe` commands: + +**Before:** +```bash +ci-helper flashpipe --update \ + --flashpipe-config ./flashpipe.yml \ + --deploy-config ./deploy-config.yml +``` + +**After (using standard Flashpipe --config flag):** +```bash +flashpipe orchestrator --update \ + --config $HOME/flashpipe.yaml \ + --deploy-config ./deploy-config.yml +``` + +**Note:** The `--config` flag works exactly like it does for all other Flashpipe commands (`deploy`, `update artifact`, etc.). If you're already using Flashpipe with a config file, the orchestrator will use the same file automatically. + +### Step 3: Update Config Files (Optional) + +If you used a separate `flashpipe-config.yml`, you can: + +**Option A:** Migrate to `$HOME/flashpipe.yaml` (recommended): +```bash +cp flashpipe-config.yml $HOME/flashpipe.yaml +# Edit to use Flashpipe flag naming conventions +``` + +**Option B:** Use command-line flags: +```bash +flashpipe orchestrator --update \ + --tmn-host tenant.hana.ondemand.com \ + --oauth-host tenant.authentication.sap.hana.ondemand.com \ + --oauth-clientid $CLIENT_ID \ + --oauth-clientsecret $CLIENT_SECRET \ + --deploy-config ./deploy-config.yml +``` + +### Step 4: Test the Migration + +Run a test deployment to a non-production environment: + +```bash +flashpipe orchestrator --update-only \ + --deployment-prefix TEST \ + --deploy-config ./deploy-config.yml \ + --packages-dir ./packages \ + --debug +``` + +Review the logs to ensure everything works as expected. + +### Step 5: Remove Standalone CLI + +Once migration is complete and tested: + +```bash +# Remove the old ci-helper binary +rm /usr/local/bin/ci-helper # or wherever it was installed + +# Remove old config files if migrated +rm ./flashpipe-config.yml # if you migrated to flashpipe.yaml +``` + +## CI/CD Pipeline Examples + +### GitHub Actions + +**Before:** +```yaml +- name: Deploy with ci-helper + run: | + ci-helper flashpipe --update \ + --flashpipe-config ./flashpipe.yml \ + --deploy-config ./configs/dev.yml \ + --deployment-prefix DEV +``` + +**After (recommended - using secrets in config file):** +```yaml +- name: Deploy with Flashpipe Orchestrator + run: | + # Create config file from secrets + echo "tmn-host: ${{ secrets.CPI_TMN_HOST }}" > flashpipe.yaml + echo "oauth-host: ${{ secrets.CPI_OAUTH_HOST }}" >> flashpipe.yaml + echo "oauth-clientid: ${{ secrets.CPI_CLIENT_ID }}" >> flashpipe.yaml + echo "oauth-clientsecret: ${{ secrets.CPI_CLIENT_SECRET }}" >> flashpipe.yaml + + flashpipe orchestrator --update \ + --config ./flashpipe.yaml \ + --deploy-config ./configs/dev.yml \ + --deployment-prefix DEV +``` + +**Alternative (using individual flags):** +```yaml +- name: Deploy with Flashpipe Orchestrator + run: | + flashpipe orchestrator --update \ + --deploy-config ./configs/dev.yml \ + --deployment-prefix DEV \ + --tmn-host ${{ secrets.CPI_TMN_HOST }} \ + --oauth-host ${{ secrets.CPI_OAUTH_HOST }} \ + --oauth-clientid ${{ secrets.CPI_CLIENT_ID }} \ + --oauth-clientsecret ${{ secrets.CPI_CLIENT_SECRET }} +``` + +### Azure DevOps + +**Before:** +```yaml +- task: Bash@3 + inputs: + script: | + ci-helper flashpipe --update \ + --flashpipe-config ./flashpipe.yml \ + --deploy-config ./deploy-config.yml +``` + +**After (recommended - using config file):** +```yaml +- task: Bash@3 + inputs: + script: | + # Create config file from pipeline variables + cat > flashpipe.yaml < flashpipe.yaml < 0 { + log.Debug().Msgf("Retrieved %d string parameters in this batch (progress: %d/%d)", batchCount, len(allParameters), totalCount) + } else { + log.Debug().Msgf("Retrieved %d string parameters in this batch (total so far: %d)", batchCount, len(allParameters)) + } + + // If we got fewer results than batch size, we've reached the end + if batchCount < batchSize { + break + } + + skip += batchSize + } + + log.Info().Msgf("Retrieved %d total string parameters", len(allParameters)) + return allParameters, nil +} + +// GetBinaryParameters retrieves all binary parameters from partner directory +func (pd *PartnerDirectory) GetBinaryParameters(selectFields string) ([]BinaryParameter, error) { + basePath := "/api/v1/BinaryParameters" + separator := "?" + if selectFields != "" { + basePath += "?$select=" + url.QueryEscape(selectFields) + separator = "&" + } + + allParameters := []BinaryParameter{} + skip := 0 + batchSize := 30 // Binary parameters API has a lower limit than string parameters + totalCount := -1 + + for { + path := fmt.Sprintf("%s%s$inlinecount=allpages&$top=%d&$skip=%d", basePath, separator, batchSize, skip) + log.Debug().Msgf("Getting binary parameters from %s", path) + + resp, err := pd.exe.ExecGetRequest(path, map[string]string{ + "Accept": "application/json", + }) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("get binary parameters failed with response code = %d", resp.StatusCode) + } + + body, err := pd.exe.ReadRespBody(resp) + if err != nil { + return nil, err + } + + var result struct { + D struct { + Results []BinaryParameter `json:"results"` + Count string `json:"__count"` + } `json:"d"` + } + + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + // Parse total count on first iteration + if totalCount == -1 && result.D.Count != "" { + fmt.Sscanf(result.D.Count, "%d", &totalCount) + log.Info().Msgf("Total binary parameters available: %d", totalCount) + } + + batchCount := len(result.D.Results) + allParameters = append(allParameters, result.D.Results...) + + if totalCount > 0 { + log.Debug().Msgf("Retrieved %d binary parameters in this batch (progress: %d/%d)", batchCount, len(allParameters), totalCount) + } else { + log.Debug().Msgf("Retrieved %d binary parameters in this batch (total so far: %d)", batchCount, len(allParameters)) + } + + // If we got fewer results than batch size, we've reached the end + if batchCount < batchSize { + break + } + + skip += batchSize + } + + log.Info().Msgf("Retrieved %d total binary parameters", len(allParameters)) + return allParameters, nil +} + +// GetStringParameter retrieves a single string parameter +func (pd *PartnerDirectory) GetStringParameter(pid, id string) (*StringParameter, error) { + path := fmt.Sprintf("/api/v1/StringParameters(Pid='%s',Id='%s')", + url.QueryEscape(pid), + url.QueryEscape(id)) + + log.Debug().Msgf("Getting string parameter %s/%s", pid, id) + + resp, err := pd.exe.ExecGetRequest(path, map[string]string{ + "Accept": "application/json", + }) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("get string parameter failed with response code = %d", resp.StatusCode) + } + + body, err := pd.exe.ReadRespBody(resp) + if err != nil { + return nil, err + } + + var result struct { + D StringParameter `json:"d"` + } + + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + return &result.D, nil +} + +// GetBinaryParameter retrieves a single binary parameter +func (pd *PartnerDirectory) GetBinaryParameter(pid, id string) (*BinaryParameter, error) { + path := fmt.Sprintf("/api/v1/BinaryParameters(Pid='%s',Id='%s')", + url.QueryEscape(pid), + url.QueryEscape(id)) + + log.Debug().Msgf("Getting binary parameter %s/%s", pid, id) + + resp, err := pd.exe.ExecGetRequest(path, map[string]string{ + "Accept": "application/json", + }) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("get binary parameter failed with response code = %d", resp.StatusCode) + } + + body, err := pd.exe.ReadRespBody(resp) + if err != nil { + return nil, err + } + + var result struct { + D BinaryParameter `json:"d"` + } + + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + return &result.D, nil +} + +// CreateStringParameter creates a new string parameter +func (pd *PartnerDirectory) CreateStringParameter(param StringParameter) error { + body := map[string]string{ + "Pid": param.Pid, + "Id": param.ID, + "Value": param.Value, + } + + bodyJSON, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("failed to marshal body: %w", err) + } + + log.Debug().Msgf("Creating string parameter %s/%s", param.Pid, param.ID) + + resp, err := pd.exe.ExecRequestWithCookies("POST", "/api/v1/StringParameters", + bytes.NewReader(bodyJSON), map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + }, nil) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusCreated { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("create string parameter failed with response code = %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil +} + +// UpdateStringParameter updates an existing string parameter +func (pd *PartnerDirectory) UpdateStringParameter(param StringParameter) error { + body := map[string]string{"Value": param.Value} + + bodyJSON, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("failed to marshal body: %w", err) + } + + path := fmt.Sprintf("/api/v1/StringParameters(Pid='%s',Id='%s')", + url.QueryEscape(param.Pid), + url.QueryEscape(param.ID)) + + log.Debug().Msgf("Updating string parameter %s/%s", param.Pid, param.ID) + + resp, err := pd.exe.ExecRequestWithCookies("PUT", path, + bytes.NewReader(bodyJSON), map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + }, nil) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("update string parameter failed with response code = %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil +} + +// DeleteStringParameter deletes a string parameter +func (pd *PartnerDirectory) DeleteStringParameter(pid, id string) error { + path := fmt.Sprintf("/api/v1/StringParameters(Pid='%s',Id='%s')", + url.QueryEscape(pid), + url.QueryEscape(id)) + + log.Debug().Msgf("Deleting string parameter %s/%s", pid, id) + + resp, err := pd.exe.ExecRequestWithCookies("DELETE", path, nil, map[string]string{ + "Accept": "application/json", + }, nil) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("delete string parameter failed with response code = %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil +} + +// CreateBinaryParameter creates a new binary parameter +func (pd *PartnerDirectory) CreateBinaryParameter(param BinaryParameter) error { + body := map[string]string{ + "Pid": param.Pid, + "Id": param.ID, + "Value": param.Value, + "ContentType": param.ContentType, + } + + bodyJSON, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("failed to marshal body: %w", err) + } + + log.Debug().Msgf("Creating binary parameter %s/%s", param.Pid, param.ID) + + resp, err := pd.exe.ExecRequestWithCookies("POST", "/api/v1/BinaryParameters", + bytes.NewReader(bodyJSON), map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + }, nil) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusCreated { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("create binary parameter failed with response code = %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil +} + +// UpdateBinaryParameter updates an existing binary parameter +func (pd *PartnerDirectory) UpdateBinaryParameter(param BinaryParameter) error { + body := map[string]string{ + "Value": param.Value, + "ContentType": param.ContentType, + } + + bodyJSON, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("failed to marshal body: %w", err) + } + + path := fmt.Sprintf("/api/v1/BinaryParameters(Pid='%s',Id='%s')", + url.QueryEscape(param.Pid), + url.QueryEscape(param.ID)) + + log.Debug().Msgf("Updating binary parameter %s/%s", param.Pid, param.ID) + + resp, err := pd.exe.ExecRequestWithCookies("PUT", path, + bytes.NewReader(bodyJSON), map[string]string{ + "Content-Type": "application/json", + "Accept": "application/json", + }, nil) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("update binary parameter failed with response code = %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil +} + +// DeleteBinaryParameter deletes a binary parameter +func (pd *PartnerDirectory) DeleteBinaryParameter(pid, id string) error { + path := fmt.Sprintf("/api/v1/BinaryParameters(Pid='%s',Id='%s')", + url.QueryEscape(pid), + url.QueryEscape(id)) + + log.Debug().Msgf("Deleting binary parameter %s/%s", pid, id) + + resp, err := pd.exe.ExecRequestWithCookies("DELETE", path, nil, map[string]string{ + "Accept": "application/json", + }, nil) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("delete binary parameter failed with response code = %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil +} + +// BatchSyncStringParameters syncs string parameters using batch operations +func (pd *PartnerDirectory) BatchSyncStringParameters(params []StringParameter, batchSize int) (*BatchResult, error) { + if batchSize <= 0 { + batchSize = DefaultBatchSize + } + + results := &BatchResult{ + Created: []string{}, + Updated: []string{}, + Unchanged: []string{}, + Errors: []string{}, + } + + // Process in batches + for i := 0; i < len(params); i += batchSize { + end := i + batchSize + if end > len(params) { + end = len(params) + } + + batchParams := params[i:end] + log.Debug().Msgf("Processing string parameter batch %d-%d of %d", i+1, end, len(params)) + + // Create batch request + batch := pd.exe.NewBatchRequest() + + // Check each parameter and add appropriate operation + for idx, param := range batchParams { + contentID := fmt.Sprintf("%d", idx+1) + key := fmt.Sprintf("%s/%s", param.Pid, param.ID) + + // Check if parameter exists + existing, err := pd.GetStringParameter(param.Pid, param.ID) + if err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + continue + } + + if existing == nil { + // Create new parameter + httpclnt.AddCreateStringParameterOp(batch, param.Pid, param.ID, param.Value, contentID) + } else if existing.Value != param.Value { + // Update existing parameter + httpclnt.AddUpdateStringParameterOp(batch, param.Pid, param.ID, param.Value, contentID) + } else { + // Unchanged + results.Unchanged = append(results.Unchanged, key) + continue + } + } + + // Execute batch + resp, err := batch.Execute() + if err == nil && len(resp.Operations) > 0 { + // Process responses + for idx, opResp := range resp.Operations { + if idx >= len(batchParams) { + break + } + param := batchParams[idx] + key := fmt.Sprintf("%s/%s", param.Pid, param.ID) + + if opResp.Error != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, opResp.Error)) + } else if opResp.StatusCode >= 200 && opResp.StatusCode < 300 { + // Check if it was a create or update based on status code + if opResp.StatusCode == http.StatusCreated || opResp.StatusCode == 201 { + results.Created = append(results.Created, key) + } else { + results.Updated = append(results.Updated, key) + } + } else { + results.Errors = append(results.Errors, fmt.Sprintf("%s: HTTP %d", key, opResp.StatusCode)) + } + } + } else if err != nil { + return nil, fmt.Errorf("batch execution failed: %w", err) + } + } + + return results, nil +} + +// BatchSyncBinaryParameters syncs binary parameters using batch operations +func (pd *PartnerDirectory) BatchSyncBinaryParameters(params []BinaryParameter, batchSize int) (*BatchResult, error) { + if batchSize <= 0 { + batchSize = DefaultBatchSize + } + + results := &BatchResult{ + Created: []string{}, + Updated: []string{}, + Unchanged: []string{}, + Errors: []string{}, + } + + // Process in batches + for i := 0; i < len(params); i += batchSize { + end := i + batchSize + if end > len(params) { + end = len(params) + } + + batchParams := params[i:end] + log.Debug().Msgf("Processing binary parameter batch %d-%d of %d", i+1, end, len(params)) + + // Create batch request + batch := pd.exe.NewBatchRequest() + + // Check each parameter and add appropriate operation + for idx, param := range batchParams { + contentID := fmt.Sprintf("%d", idx+1) + key := fmt.Sprintf("%s/%s", param.Pid, param.ID) + + // Check if parameter exists + existing, err := pd.GetBinaryParameter(param.Pid, param.ID) + if err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + continue + } + + if existing == nil { + // Create new parameter + httpclnt.AddCreateBinaryParameterOp(batch, param.Pid, param.ID, param.Value, param.ContentType, contentID) + } else if existing.Value != param.Value || existing.ContentType != param.ContentType { + // Update existing parameter + httpclnt.AddUpdateBinaryParameterOp(batch, param.Pid, param.ID, param.Value, param.ContentType, contentID) + } else { + // Unchanged + results.Unchanged = append(results.Unchanged, key) + continue + } + } + + // Execute batch + resp, err := batch.Execute() + if err == nil && len(resp.Operations) > 0 { + // Process responses + for idx, opResp := range resp.Operations { + if idx >= len(batchParams) { + break + } + param := batchParams[idx] + key := fmt.Sprintf("%s/%s", param.Pid, param.ID) + + if opResp.Error != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, opResp.Error)) + } else if opResp.StatusCode >= 200 && opResp.StatusCode < 300 { + // Check if it was a create or update based on status code + if opResp.StatusCode == http.StatusCreated || opResp.StatusCode == 201 { + results.Created = append(results.Created, key) + } else { + results.Updated = append(results.Updated, key) + } + } else { + results.Errors = append(results.Errors, fmt.Sprintf("%s: HTTP %d", key, opResp.StatusCode)) + } + } + } else if err != nil { + return nil, fmt.Errorf("batch execution failed: %w", err) + } + } + + return results, nil +} + +// BatchDeleteStringParameters deletes string parameters using batch operations +func (pd *PartnerDirectory) BatchDeleteStringParameters(pidsToDelete []struct{ Pid, ID string }, batchSize int) (*BatchResult, error) { + if batchSize <= 0 { + batchSize = DefaultBatchSize + } + + results := &BatchResult{ + Deleted: []string{}, + Errors: []string{}, + } + + // Process in batches + for i := 0; i < len(pidsToDelete); i += batchSize { + end := i + batchSize + if end > len(pidsToDelete) { + end = len(pidsToDelete) + } + + batchItems := pidsToDelete[i:end] + log.Debug().Msgf("Processing string parameter deletion batch %d-%d of %d", i+1, end, len(pidsToDelete)) + + // Create batch request + batch := pd.exe.NewBatchRequest() + + for idx, item := range batchItems { + contentID := fmt.Sprintf("%d", idx+1) + httpclnt.AddDeleteStringParameterOp(batch, item.Pid, item.ID, contentID) + } + + // Execute batch + resp, err := batch.Execute() + if err != nil { + return nil, fmt.Errorf("batch deletion failed: %w", err) + } + + // Process responses + for idx, opResp := range resp.Operations { + if idx >= len(batchItems) { + break + } + item := batchItems[idx] + key := fmt.Sprintf("%s/%s", item.Pid, item.ID) + + if opResp.Error != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, opResp.Error)) + } else if opResp.StatusCode >= 200 && opResp.StatusCode < 300 { + results.Deleted = append(results.Deleted, key) + } else { + results.Errors = append(results.Errors, fmt.Sprintf("%s: HTTP %d", key, opResp.StatusCode)) + } + } + } + + return results, nil +} + +// BatchDeleteBinaryParameters deletes binary parameters using batch operations +func (pd *PartnerDirectory) BatchDeleteBinaryParameters(pidsToDelete []struct{ Pid, ID string }, batchSize int) (*BatchResult, error) { + if batchSize <= 0 { + batchSize = DefaultBatchSize + } + + results := &BatchResult{ + Deleted: []string{}, + Errors: []string{}, + } + + // Process in batches + for i := 0; i < len(pidsToDelete); i += batchSize { + end := i + batchSize + if end > len(pidsToDelete) { + end = len(pidsToDelete) + } + + batchItems := pidsToDelete[i:end] + log.Debug().Msgf("Processing binary parameter deletion batch %d-%d of %d", i+1, end, len(pidsToDelete)) + + // Create batch request + batch := pd.exe.NewBatchRequest() + + for idx, item := range batchItems { + contentID := fmt.Sprintf("%d", idx+1) + httpclnt.AddDeleteBinaryParameterOp(batch, item.Pid, item.ID, contentID) + } + + // Execute batch + resp, err := batch.Execute() + if err != nil { + return nil, fmt.Errorf("batch deletion failed: %w", err) + } + + // Process responses + for idx, opResp := range resp.Operations { + if idx >= len(batchItems) { + break + } + item := batchItems[idx] + key := fmt.Sprintf("%s/%s", item.Pid, item.ID) + + if opResp.Error != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, opResp.Error)) + } else if opResp.StatusCode >= 200 && opResp.StatusCode < 300 { + results.Deleted = append(results.Deleted, key) + } else { + results.Errors = append(results.Errors, fmt.Sprintf("%s: HTTP %d", key, opResp.StatusCode)) + } + } + } + + return results, nil +} diff --git a/internal/cmd/apiproduct.go b/internal/cmd/apiproduct.go index a32082e..15fff25 100644 --- a/internal/cmd/apiproduct.go +++ b/internal/cmd/apiproduct.go @@ -23,15 +23,19 @@ func NewAPIProductCommand() *cobra.Command { Use: "apiproduct", Short: "Sync API Management products between tenant and Git", Long: `Synchronise API Management products between SAP Integration Suite -tenant and a Git repository.`, +tenant and a Git repository. + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'sync.apiproduct' section. CLI flags override config file settings.`, PreRunE: func(cmd *cobra.Command, args []string) error { // If artifacts directory is provided, validate that is it a subdirectory of Git repo - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "sync.apiproduct.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } if gitRepoDir != "" { - artifactsDir, err := config.GetStringWithEnvExpand(cmd, "dir-artifacts") + artifactsDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-artifacts", "sync.apiproduct.dirArtifacts") if err != nil { return fmt.Errorf("security alert for --dir-artifacts: %w", err) } @@ -41,7 +45,7 @@ tenant and a Git repository.`, } } // Validate target - target := config.GetString(cmd, "target") + target := config.GetStringWithFallback(cmd, "target", "sync.apiproduct.target") switch target { case "git", "tenant": default: @@ -65,25 +69,26 @@ tenant and a Git repository.`, func runSyncAPIProduct(cmd *cobra.Command) error { log.Info().Msg("Executing sync apiproduct command") - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + // Support reading from config file under 'sync.apiproduct' key + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "sync.apiproduct.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } - artifactsDir, err := config.GetStringWithEnvExpandWithDefault(cmd, "dir-artifacts", gitRepoDir) - if err != nil { - return fmt.Errorf("security alert for --dir-artifacts: %w", err) + artifactsDir := config.GetStringWithFallback(cmd, "dir-artifacts", "sync.apiproduct.dirArtifacts") + if artifactsDir == "" { + artifactsDir = gitRepoDir } - workDir, err := config.GetStringWithEnvExpand(cmd, "dir-work") + workDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-work", "sync.apiproduct.dirWork") if err != nil { return fmt.Errorf("security alert for --dir-work: %w", err) } - includedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-include")) - excludedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-exclude")) - commitMsg := config.GetString(cmd, "git-commit-msg") - commitUser := config.GetString(cmd, "git-commit-user") - commitEmail := config.GetString(cmd, "git-commit-email") - skipCommit := config.GetBool(cmd, "git-skip-commit") - target := config.GetString(cmd, "target") + includedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-include", "sync.apiproduct.idsInclude")) + excludedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-exclude", "sync.apiproduct.idsExclude")) + commitMsg := config.GetStringWithFallback(cmd, "git-commit-msg", "sync.apiproduct.gitCommitMsg") + commitUser := config.GetStringWithFallback(cmd, "git-commit-user", "sync.apiproduct.gitCommitUser") + commitEmail := config.GetStringWithFallback(cmd, "git-commit-email", "sync.apiproduct.gitCommitEmail") + skipCommit := config.GetBoolWithFallback(cmd, "git-skip-commit", "sync.apiproduct.gitSkipCommit") + target := config.GetStringWithFallback(cmd, "target", "sync.apiproduct.target") serviceDetails := api.GetServiceDetails(cmd) // Initialise HTTP executer diff --git a/internal/cmd/apiproxy.go b/internal/cmd/apiproxy.go index b131a6b..c8f5fed 100644 --- a/internal/cmd/apiproxy.go +++ b/internal/cmd/apiproxy.go @@ -24,15 +24,19 @@ func NewAPIProxyCommand() *cobra.Command { Aliases: []string{"apim"}, Short: "Sync API Management proxies (with dependent artifacts) between tenant and Git", Long: `Synchronise API Management proxies (with dependent artifacts) between SAP Integration Suite -tenant and a Git repository.`, +tenant and a Git repository. + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'sync.apiproxy' section. CLI flags override config file settings.`, PreRunE: func(cmd *cobra.Command, args []string) error { // If artifacts directory is provided, validate that is it a subdirectory of Git repo - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "sync.apiproxy.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } if gitRepoDir != "" { - artifactsDir, err := config.GetStringWithEnvExpand(cmd, "dir-artifacts") + artifactsDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-artifacts", "sync.apiproxy.dirArtifacts") if err != nil { return fmt.Errorf("security alert for --dir-artifacts: %w", err) } @@ -42,7 +46,7 @@ tenant and a Git repository.`, } } // Validate target - target := config.GetString(cmd, "target") + target := config.GetStringWithFallback(cmd, "target", "sync.apiproxy.target") switch target { case "git", "tenant": default: @@ -66,25 +70,26 @@ tenant and a Git repository.`, func runSyncAPIProxy(cmd *cobra.Command) error { log.Info().Msg("Executing sync apiproxy command") - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + // Support reading from config file under 'sync.apiproxy' key + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "sync.apiproxy.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } - artifactsDir, err := config.GetStringWithEnvExpandWithDefault(cmd, "dir-artifacts", gitRepoDir) - if err != nil { - return fmt.Errorf("security alert for --dir-artifacts: %w", err) + artifactsDir := config.GetStringWithFallback(cmd, "dir-artifacts", "sync.apiproxy.dirArtifacts") + if artifactsDir == "" { + artifactsDir = gitRepoDir } - workDir, err := config.GetStringWithEnvExpand(cmd, "dir-work") + workDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-work", "sync.apiproxy.dirWork") if err != nil { return fmt.Errorf("security alert for --dir-work: %w", err) } - includedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-include")) - excludedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-exclude")) - commitMsg := config.GetString(cmd, "git-commit-msg") - commitUser := config.GetString(cmd, "git-commit-user") - commitEmail := config.GetString(cmd, "git-commit-email") - skipCommit := config.GetBool(cmd, "git-skip-commit") - target := config.GetString(cmd, "target") + includedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-include", "sync.apiproxy.idsInclude")) + excludedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-exclude", "sync.apiproxy.idsExclude")) + commitMsg := config.GetStringWithFallback(cmd, "git-commit-msg", "sync.apiproxy.gitCommitMsg") + commitUser := config.GetStringWithFallback(cmd, "git-commit-user", "sync.apiproxy.gitCommitUser") + commitEmail := config.GetStringWithFallback(cmd, "git-commit-email", "sync.apiproxy.gitCommitEmail") + skipCommit := config.GetBoolWithFallback(cmd, "git-skip-commit", "sync.apiproxy.gitSkipCommit") + target := config.GetStringWithFallback(cmd, "target", "sync.apiproxy.target") serviceDetails := api.GetServiceDetails(cmd) // Initialise HTTP executer diff --git a/internal/cmd/artifact.go b/internal/cmd/artifact.go index 5b1307f..7eb8c66 100644 --- a/internal/cmd/artifact.go +++ b/internal/cmd/artifact.go @@ -20,10 +20,14 @@ func NewArtifactCommand() *cobra.Command { Use: "artifact", Short: "Create/update artifacts", Long: `Create or update artifacts on the -SAP Integration Suite tenant.`, +SAP Integration Suite tenant. + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'update.artifact' section. CLI flags override config file settings.`, PreRunE: func(cmd *cobra.Command, args []string) error { // Validate the artifact type - artifactType := config.GetString(cmd, "artifact-type") + artifactType := config.GetStringWithFallback(cmd, "artifact-type", "update.artifact.artifactType") switch artifactType { case "MessageMapping", "ScriptCollection", "Integration", "ValueMapping": default: @@ -42,16 +46,17 @@ SAP Integration Suite tenant.`, } // Define cobra flags, the default value has the lowest (least significant) precedence - artifactCmd.Flags().String("artifact-id", "", "ID of artifact") - artifactCmd.Flags().String("artifact-name", "", "Name of artifact. Defaults to artifact-id value when not provided") - artifactCmd.Flags().String("package-id", "", "ID of Integration Package") - artifactCmd.Flags().String("package-name", "", "Name of Integration Package. Defaults to package-id value when not provided") - artifactCmd.Flags().String("dir-artifact", "", "Directory containing contents of designtime artifact") - artifactCmd.Flags().String("file-param", "", "Use a different parameters.prop file instead of the default in src/main/resources/ ") - artifactCmd.Flags().String("file-manifest", "", "Use a different MANIFEST.MF file instead of the default in META-INF/") - artifactCmd.Flags().String("dir-work", "/tmp", "Working directory for in-transit files") - artifactCmd.Flags().StringSlice("script-collection-map", nil, "Comma-separated source-target ID pairs for converting script collection references during create/update") - artifactCmd.Flags().String("artifact-type", "Integration", "Artifact type. Allowed values: Integration, MessageMapping, ScriptCollection, ValueMapping") + // Note: These can be set in config file under 'update.artifact' key + artifactCmd.Flags().String("artifact-id", "", "ID of artifact (config: update.artifact.artifactId)") + artifactCmd.Flags().String("artifact-name", "", "Name of artifact. Defaults to artifact-id value when not provided (config: update.artifact.artifactName)") + artifactCmd.Flags().String("package-id", "", "ID of Integration Package (config: update.artifact.packageId)") + artifactCmd.Flags().String("package-name", "", "Name of Integration Package. Defaults to package-id value when not provided (config: update.artifact.packageName)") + artifactCmd.Flags().String("dir-artifact", "", "Directory containing contents of designtime artifact (config: update.artifact.dirArtifact)") + artifactCmd.Flags().String("file-param", "", "Use a different parameters.prop file instead of the default in src/main/resources/ (config: update.artifact.fileParam)") + artifactCmd.Flags().String("file-manifest", "", "Use a different MANIFEST.MF file instead of the default in META-INF/ (config: update.artifact.fileManifest)") + artifactCmd.Flags().String("dir-work", "/tmp", "Working directory for in-transit files (config: update.artifact.dirWork)") + artifactCmd.Flags().StringSlice("script-collection-map", nil, "Comma-separated source-target ID pairs for converting script collection references during create/update (config: update.artifact.scriptCollectionMap)") + artifactCmd.Flags().String("artifact-type", "Integration", "Artifact type. Allowed values: Integration, MessageMapping, ScriptCollection, ValueMapping (config: update.artifact.artifactType)") // TODO - another flag for replacing value mapping in QAS? _ = artifactCmd.MarkFlagRequired("artifact-id") @@ -62,29 +67,30 @@ SAP Integration Suite tenant.`, } func runUpdateArtifact(cmd *cobra.Command) error { - artifactType := config.GetString(cmd, "artifact-type") + // Support reading from config file under 'update.artifact' key + artifactType := config.GetStringWithFallback(cmd, "artifact-type", "update.artifact.artifactType") log.Info().Msgf("Executing update artifact %v command", artifactType) - artifactId := config.GetString(cmd, "artifact-id") - artifactName := config.GetString(cmd, "artifact-name") - packageId := config.GetString(cmd, "package-id") - packageName := config.GetString(cmd, "package-name") + artifactId := config.GetStringWithFallback(cmd, "artifact-id", "update.artifact.artifactId") + artifactName := config.GetStringWithFallback(cmd, "artifact-name", "update.artifact.artifactName") + packageId := config.GetStringWithFallback(cmd, "package-id", "update.artifact.packageId") + packageName := config.GetStringWithFallback(cmd, "package-name", "update.artifact.packageName") // Default package name to package ID if it is not provided if packageName == "" { log.Info().Msgf("Using package ID %v as package name", packageId) packageName = packageId } - artifactDir, err := config.GetStringWithEnvExpand(cmd, "dir-artifact") + artifactDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-artifact", "update.artifact.dirArtifact") if err != nil { return fmt.Errorf("security alert for --dir-artifact: %w", err) } - parametersFile := config.GetString(cmd, "file-param") - manifestFile := config.GetString(cmd, "file-manifest") - workDir, err := config.GetStringWithEnvExpand(cmd, "dir-work") + parametersFile := config.GetStringWithFallback(cmd, "file-param", "update.artifact.fileParam") + manifestFile := config.GetStringWithFallback(cmd, "file-manifest", "update.artifact.fileManifest") + workDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-work", "update.artifact.dirWork") if err != nil { return fmt.Errorf("security alert for --dir-work: %w", err) } - scriptMap := str.TrimSlice(config.GetStringSlice(cmd, "script-collection-map")) + scriptMap := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "script-collection-map", "update.artifact.scriptCollectionMap")) defaultParamFile := fmt.Sprintf("%v/src/main/resources/parameters.prop", artifactDir) if parametersFile == "" { diff --git a/internal/cmd/config_generate.go b/internal/cmd/config_generate.go new file mode 100644 index 0000000..a51a804 --- /dev/null +++ b/internal/cmd/config_generate.go @@ -0,0 +1,571 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/engswee/flashpipe/internal/analytics" + "github.com/engswee/flashpipe/internal/config" + "github.com/engswee/flashpipe/internal/file" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +func NewConfigGenerateCommand() *cobra.Command { + + configCmd := &cobra.Command{ + Use: "config-generate", + Short: "Generate or update deployment configuration", + Long: `Generate or update deployment configuration from package directory structure. + +This command scans the packages directory and generates/updates a deployment configuration +file (001-deploy-config.yml) with all discovered packages and artifacts. + +Features: + - Extracts package metadata from {PackageName}.json files + - Extracts artifact display names from MANIFEST.MF (Bundle-Name) + - Extracts artifact types from MANIFEST.MF (SAP-BundleType) + - Preserves existing configuration settings (sync/deploy flags, config overrides) + - Smart merging of new and existing configurations + - Filter by specific packages or artifacts`, + Example: ` # Generate config with defaults + flashpipe config-generate + + # Specify custom directories + flashpipe config-generate --packages-dir ./my-packages --output ./my-config.yml + + # Generate config for specific packages only + flashpipe config-generate --package-filter "DeviceManagement,GenericPipeline" + + # Generate config for specific artifacts only + flashpipe config-generate --artifact-filter "MDMEquipmentMutationOutbound,GenericBroadcaster" + + # Combine package and artifact filters + flashpipe config-generate --package-filter "DeviceManagement" --artifact-filter "MDMEquipmentMutationOutbound"`, + RunE: func(cmd *cobra.Command, args []string) (err error) { + startTime := time.Now() + if err = runConfigGenerate(cmd); err != nil { + cmd.SilenceUsage = true + } + analytics.Log(cmd, err, startTime) + return + }, + } + + configCmd.Flags().String("packages-dir", "./packages", + "Path to packages directory") + configCmd.Flags().String("output", "./001-deploy-config.yml", + "Path to output configuration file") + configCmd.Flags().StringSlice("package-filter", nil, + "Comma separated list of packages to include (e.g., 'Package1,Package2')") + configCmd.Flags().StringSlice("artifact-filter", nil, + "Comma separated list of artifacts to include (e.g., 'Artifact1,Artifact2')") + + return configCmd +} + +func runConfigGenerate(cmd *cobra.Command) error { + packagesDir := config.GetString(cmd, "packages-dir") + outputFile := config.GetString(cmd, "output") + packageFilter := config.GetStringSlice(cmd, "package-filter") + artifactFilter := config.GetStringSlice(cmd, "artifact-filter") + + generator := NewConfigGenerator(packagesDir, outputFile, packageFilter, artifactFilter) + + if err := generator.Generate(); err != nil { + return err + } + + return nil +} + +// ConfigGenerator handles configuration generation +type ConfigGenerator struct { + PackagesDir string + OutputFile string + PackageFilter []string + ArtifactFilter []string + ExistingConfig *DeployConfig + Stats GenerationStats +} + +// GenerationStats tracks generation statistics +type GenerationStats struct { + PackagesPreserved int + PackagesAdded int + PackagesRemoved int + PackagesFiltered int + PackagePropertiesExtracted int + PackagePropertiesPreserved int + ArtifactsPreserved int + ArtifactsAdded int + ArtifactsRemoved int + ArtifactsFiltered int + ArtifactsNameExtracted int + ArtifactsNamePreserved int + ArtifactsTypeExtracted int + ArtifactsTypePreserved int +} + +// DeployConfig represents the complete deployment configuration +type DeployConfig struct { + DeploymentPrefix string `yaml:"deploymentPrefix,omitempty"` + Packages []Package `yaml:"packages"` +} + +// Package represents a SAP CPI package +type Package struct { + ID string `yaml:"integrationSuiteId"` + PackageDir string `yaml:"packageDir,omitempty"` + DisplayName string `yaml:"displayName,omitempty"` + Description string `yaml:"description,omitempty"` + ShortText string `yaml:"short_text,omitempty"` + Sync bool `yaml:"sync"` + Deploy bool `yaml:"deploy"` + Artifacts []Artifact `yaml:"artifacts"` +} + +// Artifact represents a SAP CPI artifact +type Artifact struct { + Id string `yaml:"artifactId"` + ArtifactDir string `yaml:"artifactDir"` + DisplayName string `yaml:"displayName,omitempty"` + Type string `yaml:"type"` + Sync bool `yaml:"sync"` + Deploy bool `yaml:"deploy"` + ConfigOverrides map[string]interface{} `yaml:"configOverrides,omitempty"` +} + +// PackageMetadata represents metadata from package JSON +type PackageMetadata struct { + ID string `json:"Id"` + Name string `json:"Name"` + Description string `json:"Description"` + ShortText string `json:"ShortText"` +} + +// NewConfigGenerator creates a new configuration generator +func NewConfigGenerator(packagesDir, outputFile string, packageFilter, artifactFilter []string) *ConfigGenerator { + return &ConfigGenerator{ + PackagesDir: packagesDir, + OutputFile: outputFile, + PackageFilter: packageFilter, + ArtifactFilter: artifactFilter, + } +} + +// shouldIncludePackage checks if a package should be included based on filter +func (g *ConfigGenerator) shouldIncludePackage(packageName string) bool { + if len(g.PackageFilter) == 0 { + return true + } + for _, filterPkg := range g.PackageFilter { + if filterPkg == packageName { + return true + } + } + return false +} + +// shouldIncludeArtifact checks if an artifact should be included based on filter +func (g *ConfigGenerator) shouldIncludeArtifact(artifactName string) bool { + if len(g.ArtifactFilter) == 0 { + return true + } + for _, filterArt := range g.ArtifactFilter { + if filterArt == artifactName { + return true + } + } + return false +} + +// Generate generates or updates the deployment configuration +func (g *ConfigGenerator) Generate() error { + log.Info().Msg("Generating/Updating Configuration") + log.Info().Msgf("Packages directory: %s", g.PackagesDir) + log.Info().Msgf("Config file: %s", g.OutputFile) + + if len(g.PackageFilter) > 0 { + log.Info().Msgf("Package filter: %s", strings.Join(g.PackageFilter, ", ")) + } + if len(g.ArtifactFilter) > 0 { + log.Info().Msgf("Artifact filter: %s", strings.Join(g.ArtifactFilter, ", ")) + } + + // Check if packages directory exists + if _, err := os.Stat(g.PackagesDir); os.IsNotExist(err) { + return fmt.Errorf("packages directory '%s' not found", g.PackagesDir) + } + + // Load existing config if it exists + if _, err := os.Stat(g.OutputFile); err == nil { + log.Info().Msg("Loading existing configuration...") + data, err := os.ReadFile(g.OutputFile) + if err != nil { + return fmt.Errorf("failed to read existing config: %w", err) + } + var existingConfig DeployConfig + if err := yaml.Unmarshal(data, &existingConfig); err != nil { + return fmt.Errorf("failed to parse existing config: %w", err) + } + g.ExistingConfig = &existingConfig + } + + // Create new config structure + newConfig := DeployConfig{ + DeploymentPrefix: "", + Packages: []Package{}, + } + + // Preserve deployment prefix if exists + if g.ExistingConfig != nil { + newConfig.DeploymentPrefix = g.ExistingConfig.DeploymentPrefix + } + + // Build map of existing packages and artifacts for quick lookup + existingPackages := make(map[string]Package) + existingArtifacts := make(map[string]map[string]Artifact) + + if g.ExistingConfig != nil { + for _, pkg := range g.ExistingConfig.Packages { + existingPackages[pkg.ID] = pkg + existingArtifacts[pkg.ID] = make(map[string]Artifact) + for _, art := range pkg.Artifacts { + existingArtifacts[pkg.ID][art.Id] = art + } + } + } + + // Scan packages directory + entries, err := os.ReadDir(g.PackagesDir) + if err != nil { + return fmt.Errorf("failed to read packages directory: %w", err) + } + + processedPackages := make(map[string]bool) + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + packageName := entry.Name() + + // Apply package filter + if !g.shouldIncludePackage(packageName) { + g.Stats.PackagesFiltered++ + continue + } + + packageDir := filepath.Join(g.PackagesDir, packageName) + + log.Debug().Msgf("Processing package: %s", packageName) + + processedPackages[packageName] = true + + // Extract package metadata + metadata := g.extractPackageMetadata(packageDir, packageName) + + // Check if package exists in old config + var pkg Package + if existingPkg, exists := existingPackages[packageName]; exists { + pkg = existingPkg + g.Stats.PackagesPreserved++ + + if metadata != nil { + if pkg.PackageDir == "" || pkg.DisplayName == "" { + g.Stats.PackagePropertiesExtracted++ + } else { + g.Stats.PackagePropertiesPreserved++ + } + + if pkg.PackageDir == "" { + pkg.PackageDir = metadata.ID + } + if pkg.DisplayName == "" { + pkg.DisplayName = metadata.Name + } + if pkg.Description == "" { + pkg.Description = metadata.Description + } + if pkg.ShortText == "" { + pkg.ShortText = metadata.ShortText + } + } + } else { + pkg = Package{ + ID: packageName, + Sync: true, + Deploy: true, + } + + if metadata != nil { + pkg.PackageDir = metadata.ID + pkg.DisplayName = metadata.Name + pkg.Description = metadata.Description + pkg.ShortText = metadata.ShortText + g.Stats.PackagePropertiesExtracted++ + } + + g.Stats.PackagesAdded++ + } + + // Reset artifacts slice + pkg.Artifacts = []Artifact{} + + // Scan artifacts + artifactEntries, err := os.ReadDir(packageDir) + if err != nil { + log.Warn().Msgf("Failed to read package directory: %v", err) + continue + } + + processedArtifacts := make(map[string]bool) + + for _, artEntry := range artifactEntries { + if !artEntry.IsDir() { + continue + } + + artifactName := artEntry.Name() + + // Apply artifact filter + if !g.shouldIncludeArtifact(artifactName) { + g.Stats.ArtifactsFiltered++ + continue + } + + artifactDir := filepath.Join(packageDir, artifactName) + + processedArtifacts[artifactName] = true + + // Extract artifact metadata from MANIFEST.MF + bundleName, artifactType := g.extractManifestMetadata(artifactDir) + + // Check if artifact exists in old config + var artifact Artifact + if existingArtMap, pkgExists := existingArtifacts[packageName]; pkgExists { + if existingArt, artExists := existingArtMap[artifactName]; artExists { + artifact = existingArt + g.Stats.ArtifactsPreserved++ + + if bundleName != "" { + if artifact.DisplayName == "" { + g.Stats.ArtifactsNameExtracted++ + artifact.DisplayName = bundleName + } else { + g.Stats.ArtifactsNamePreserved++ + } + } + + if artifactType != "" { + if artifact.Type == "" { + g.Stats.ArtifactsTypeExtracted++ + artifact.Type = artifactType + } else { + g.Stats.ArtifactsTypePreserved++ + } + } + + if artifact.ArtifactDir == "" { + artifact.ArtifactDir = artifactName + } + } else { + artifact = Artifact{ + Id: artifactName, + ArtifactDir: artifactName, + DisplayName: bundleName, + Type: artifactType, + Sync: true, + Deploy: true, + ConfigOverrides: make(map[string]interface{}), + } + + if bundleName != "" { + g.Stats.ArtifactsNameExtracted++ + } + if artifactType != "" { + g.Stats.ArtifactsTypeExtracted++ + } + + g.Stats.ArtifactsAdded++ + } + } else { + artifact = Artifact{ + Id: artifactName, + ArtifactDir: artifactName, + DisplayName: bundleName, + Type: artifactType, + Sync: true, + Deploy: true, + ConfigOverrides: make(map[string]interface{}), + } + + if bundleName != "" { + g.Stats.ArtifactsNameExtracted++ + } + if artifactType != "" { + g.Stats.ArtifactsTypeExtracted++ + } + + g.Stats.ArtifactsAdded++ + } + + pkg.Artifacts = append(pkg.Artifacts, artifact) + } + + // Count removed artifacts + if existingArtMap, pkgExists := existingArtifacts[packageName]; pkgExists { + for artName := range existingArtMap { + if !processedArtifacts[artName] { + g.Stats.ArtifactsRemoved++ + } + } + } + + // Only add package if it has artifacts (when artifact filter is used) + if len(g.ArtifactFilter) > 0 && len(pkg.Artifacts) == 0 { + continue + } + + newConfig.Packages = append(newConfig.Packages, pkg) + } + + // Count removed packages + if g.ExistingConfig != nil { + for _, pkg := range g.ExistingConfig.Packages { + if !processedPackages[pkg.ID] { + g.Stats.PackagesRemoved++ + } + } + } + + // Sort packages by ID for consistency + sort.Slice(newConfig.Packages, func(i, j int) bool { + return newConfig.Packages[i].ID < newConfig.Packages[j].ID + }) + + // Write config file + if err := g.writeConfigFile(g.OutputFile, &newConfig); err != nil { + return fmt.Errorf("failed to write config file: %w", err) + } + + g.printSummary() + + return nil +} + +func (g *ConfigGenerator) extractPackageMetadata(packageDir, packageName string) *PackageMetadata { + jsonFile := filepath.Join(packageDir, packageName+".json") + if _, err := os.Stat(jsonFile); os.IsNotExist(err) { + return nil + } + + data, err := os.ReadFile(jsonFile) + if err != nil { + log.Warn().Msgf("Failed to read package JSON: %v", err) + return nil + } + + var wrapper struct { + D PackageMetadata `json:"d"` + } + + if err := yaml.Unmarshal(data, &wrapper); err != nil { + log.Warn().Msgf("Failed to parse package JSON: %v", err) + return nil + } + + return &wrapper.D +} + +func (g *ConfigGenerator) extractManifestMetadata(artifactDir string) (bundleName, artifactType string) { + manifestPath := filepath.Join(artifactDir, "META-INF", "MANIFEST.MF") + if _, err := os.Stat(manifestPath); os.IsNotExist(err) { + return "", "" + } + + manifestData, err := file.ReadManifest(manifestPath) + if err != nil { + log.Warn().Msgf("Failed to read manifest: %v", err) + return "", "" + } + + bundleName = manifestData["Bundle-Name"] + artifactType = manifestData["SAP-BundleType"] + + return bundleName, artifactType +} + +func (g *ConfigGenerator) writeConfigFile(outputPath string, cfg *DeployConfig) error { + data, err := yaml.Marshal(cfg) + if err != nil { + return err + } + + header := `# SAP CPI Deployment Configuration +# Generated by: flashpipe config-generate +# +# ============================================================================ +# FIELD DESCRIPTIONS +# ============================================================================ +# +# PACKAGE FIELDS: +# integrationSuiteId (required): Unique ID of the integration package in SAP CPI +# packageDir (required): Local directory containing the package artifacts +# displayName (optional): Override the package's display name in SAP CPI +# description (optional): Override the package description +# short_text (optional): Override the package short text +# sync (default: true): Whether to update/sync this package to the tenant +# deploy (default: true): Whether to deploy this package +# artifacts: List of artifacts within this package +# +# ARTIFACT FIELDS: +# artifactId (required): Unique ID of the artifact (IFlow, Script Collection, etc.) +# artifactDir (required): Local directory path to the artifact +# displayName (optional): Override the artifact's display name in SAP CPI +# type (required): Artifact type (Integration, ScriptCollection, MessageMapping, ValueMapping) +# sync (default: true): Whether to update/sync this artifact to the tenant +# deploy (default: true): Whether to deploy/activate this artifact +# configOverrides (optional): Override parameter values from parameters.prop +# +# ============================================================================ + +` + + return os.WriteFile(outputPath, []byte(header+string(data)), 0644) +} + +func (g *ConfigGenerator) printSummary() { + log.Info().Msgf("Configuration saved to: %s", g.OutputFile) + log.Info().Msg("Summary of Changes:") + log.Info().Msg(" Packages:") + log.Info().Msgf(" - Preserved: %d", g.Stats.PackagesPreserved) + log.Info().Msgf(" - Added: %d", g.Stats.PackagesAdded) + log.Info().Msgf(" - Removed: %d", g.Stats.PackagesRemoved) + if g.Stats.PackagesFiltered > 0 { + log.Info().Msgf(" - Filtered: %d", g.Stats.PackagesFiltered) + } + log.Info().Msg(" Package Properties (from {PackageName}.json):") + log.Info().Msgf(" - Extracted: %d", g.Stats.PackagePropertiesExtracted) + log.Info().Msgf(" - Preserved: %d", g.Stats.PackagePropertiesPreserved) + log.Info().Msg(" Artifacts:") + log.Info().Msgf(" - Preserved: %d (settings kept)", g.Stats.ArtifactsPreserved) + log.Info().Msgf(" - Added: %d (defaults applied)", g.Stats.ArtifactsAdded) + log.Info().Msgf(" - Removed: %d (deleted from config)", g.Stats.ArtifactsRemoved) + if g.Stats.ArtifactsFiltered > 0 { + log.Info().Msgf(" - Filtered: %d", g.Stats.ArtifactsFiltered) + } + log.Info().Msg(" Artifact Display Names (Bundle-Name from MANIFEST.MF):") + log.Info().Msgf(" - Extracted: %d", g.Stats.ArtifactsNameExtracted) + log.Info().Msgf(" - Preserved: %d", g.Stats.ArtifactsNamePreserved) + log.Info().Msg(" Artifact Types (SAP-BundleType from MANIFEST.MF):") + log.Info().Msgf(" - Extracted: %d", g.Stats.ArtifactsTypeExtracted) + log.Info().Msgf(" - Preserved: %d", g.Stats.ArtifactsTypePreserved) +} diff --git a/internal/cmd/configure.go b/internal/cmd/configure.go new file mode 100644 index 0000000..358a65c --- /dev/null +++ b/internal/cmd/configure.go @@ -0,0 +1,778 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/engswee/flashpipe/internal/api" + "github.com/engswee/flashpipe/internal/deploy" + "github.com/engswee/flashpipe/internal/httpclnt" + "github.com/engswee/flashpipe/internal/models" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "gopkg.in/yaml.v3" +) + +// ConfigureStats tracks configuration processing statistics +type ConfigureStats struct { + PackagesProcessed int + PackagesWithErrors int + ArtifactsProcessed int + ArtifactsConfigured int + ArtifactsDeployed int + ArtifactsFailed int + ParametersUpdated int + ParametersFailed int + BatchRequestsExecuted int + IndividualRequestsUsed int + DeploymentTasksQueued int + DeploymentTasksSuccessful int + DeploymentTasksFailed int +} + +// ConfigurationTask represents a configuration update task +type ConfigurationTask struct { + PackageID string + ArtifactID string + Version string + Parameters []models.ConfigurationParameter + UseBatch bool + BatchSize int + DisplayName string +} + +func NewConfigureCommand() *cobra.Command { + var ( + configPath string + deploymentPrefix string + packageFilter string + artifactFilter string + dryRun bool + deployRetries int + deployDelaySeconds int + parallelDeployments int + batchSize int + disableBatch bool + ) + + configureCmd := &cobra.Command{ + Use: "configure", + Short: "Configure SAP CPI artifact parameters", + Long: `Configure parameters for SAP CPI artifacts using YAML configuration files. + +This command: + - Updates configuration parameters for Integration artifacts + - Supports batch operations for efficient parameter updates + - Optionally deploys artifacts after configuration + - Two-phase operation: Configure all artifacts, then deploy if requested + - Supports deployment prefixes for multi-environment scenarios + +Configuration File Structure: + The YAML file should define packages and artifacts with their parameters: + + deploymentPrefix: "DEV_" # Optional + packages: + - integrationSuiteId: "MyPackage" + displayName: "My Integration Package" + deploy: false # Deploy all artifacts in this package after configuration + artifacts: + - artifactId: "MyFlow" + displayName: "My Integration Flow" + type: "Integration" + version: "active" # Optional, defaults to "active" + deploy: true # Deploy this specific artifact after configuration + parameters: + - key: "DatabaseURL" + value: "jdbc:mysql://localhost:3306/mydb" + - key: "MaxRetries" + value: "5" + batch: + enabled: true # Use batch operations (default: true) + batchSize: 90 # Parameters per batch (default: 90) + +Operation Modes: + 1. Configure Only: Updates parameters without deployment (default) + 2. Configure + Deploy: Updates parameters then deploys artifacts (when deploy: true) + +Batch Processing: + - By default, uses OData $batch for efficient parameter updates + - Configurable batch size (default: 90 parameters per request) + - Falls back to individual requests if batch fails + - Can be disabled globally with --disable-batch flag + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'configure' section. CLI flags override config file settings.`, + Example: ` # Configure artifacts from a config file + flashpipe configure --config-path ./config/dev-config.yml + + # Configure and deploy + flashpipe configure --config-path ./config/prod-config.yml + + # Dry run to see what would be changed + flashpipe configure --config-path ./config.yml --dry-run + + # Apply deployment prefix + flashpipe configure --config-path ./config.yml --deployment-prefix DEV_ + + # Disable batch processing + flashpipe configure --config-path ./config.yml --disable-batch`, + RunE: func(cmd *cobra.Command, args []string) error { + // Load from viper config if available (CLI flags override config file) + if !cmd.Flags().Changed("config-path") && viper.IsSet("configure.configPath") { + configPath = viper.GetString("configure.configPath") + } + if !cmd.Flags().Changed("deployment-prefix") && viper.IsSet("configure.deploymentPrefix") { + deploymentPrefix = viper.GetString("configure.deploymentPrefix") + } + if !cmd.Flags().Changed("package-filter") && viper.IsSet("configure.packageFilter") { + packageFilter = viper.GetString("configure.packageFilter") + } + if !cmd.Flags().Changed("artifact-filter") && viper.IsSet("configure.artifactFilter") { + artifactFilter = viper.GetString("configure.artifactFilter") + } + if !cmd.Flags().Changed("dry-run") && viper.IsSet("configure.dryRun") { + dryRun = viper.GetBool("configure.dryRun") + } + if !cmd.Flags().Changed("deploy-retries") && viper.IsSet("configure.deployRetries") { + deployRetries = viper.GetInt("configure.deployRetries") + } + if !cmd.Flags().Changed("deploy-delay") && viper.IsSet("configure.deployDelaySeconds") { + deployDelaySeconds = viper.GetInt("configure.deployDelaySeconds") + } + if !cmd.Flags().Changed("parallel-deployments") && viper.IsSet("configure.parallelDeployments") { + parallelDeployments = viper.GetInt("configure.parallelDeployments") + } + if !cmd.Flags().Changed("batch-size") && viper.IsSet("configure.batchSize") { + batchSize = viper.GetInt("configure.batchSize") + } + if !cmd.Flags().Changed("disable-batch") && viper.IsSet("configure.disableBatch") { + disableBatch = viper.GetBool("configure.disableBatch") + } + + // Validate required parameters + if configPath == "" { + return fmt.Errorf("--config-path is required (set via CLI flag or in config file under 'configure.configPath')") + } + + // Set defaults for deployment settings + if deployRetries == 0 { + deployRetries = 5 + } + if deployDelaySeconds == 0 { + deployDelaySeconds = 15 + } + if parallelDeployments == 0 { + parallelDeployments = 3 + } + if batchSize == 0 { + batchSize = httpclnt.DefaultBatchSize + } + + return runConfigure(cmd, configPath, deploymentPrefix, packageFilter, artifactFilter, + dryRun, deployRetries, deployDelaySeconds, parallelDeployments, batchSize, disableBatch) + }, + } + + // Flags + configureCmd.Flags().StringVarP(&configPath, "config-path", "c", "", "Path to configuration YAML file (config: configure.configPath)") + configureCmd.Flags().StringVarP(&deploymentPrefix, "deployment-prefix", "p", "", "Deployment prefix for artifact IDs (config: configure.deploymentPrefix)") + configureCmd.Flags().StringVar(&packageFilter, "package-filter", "", "Comma-separated list of packages to include (config: configure.packageFilter)") + configureCmd.Flags().StringVar(&artifactFilter, "artifact-filter", "", "Comma-separated list of artifacts to include (config: configure.artifactFilter)") + configureCmd.Flags().BoolVar(&dryRun, "dry-run", false, "Show what would be done without making changes (config: configure.dryRun)") + configureCmd.Flags().IntVar(&deployRetries, "deploy-retries", 0, "Number of retries for deployment status checks (config: configure.deployRetries, default: 5)") + configureCmd.Flags().IntVar(&deployDelaySeconds, "deploy-delay", 0, "Delay in seconds between deployment status checks (config: configure.deployDelaySeconds, default: 15)") + configureCmd.Flags().IntVar(¶llelDeployments, "parallel-deployments", 0, "Number of parallel deployments (config: configure.parallelDeployments, default: 3)") + configureCmd.Flags().IntVar(&batchSize, "batch-size", 0, "Number of parameters per batch request (config: configure.batchSize, default: 90)") + configureCmd.Flags().BoolVar(&disableBatch, "disable-batch", false, "Disable batch processing, use individual requests (config: configure.disableBatch)") + + return configureCmd +} + +func runConfigure(cmd *cobra.Command, configPath, deploymentPrefix, packageFilterStr, artifactFilterStr string, + dryRun bool, deployRetries, deployDelaySeconds, parallelDeployments, batchSize int, disableBatch bool) error { + + log.Info().Msg("Starting artifact configuration") + + // Validate deployment prefix + if deploymentPrefix != "" { + if err := deploy.ValidateDeploymentPrefix(deploymentPrefix); err != nil { + return err + } + } + + // Parse filters + packageFilter := parseFilter(packageFilterStr) + artifactFilter := parseFilter(artifactFilterStr) + + // Load configuration from file or folder + log.Info().Msgf("Loading configuration from: %s", configPath) + configFiles, err := loadConfigureConfigs(configPath) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + + log.Info().Msgf("Loaded %d configuration file(s)", len(configFiles)) + log.Info().Msgf("Deployment prefix: %s", deploymentPrefix) + log.Info().Msgf("Dry run: %v", dryRun) + log.Info().Msgf("Batch processing: %v (size: %d)", !disableBatch, batchSize) + + // Merge all configurations + configData := mergeConfigureConfigs(configFiles, deploymentPrefix) + + // Apply deployment prefix if specified + if deploymentPrefix != "" { + configData.DeploymentPrefix = deploymentPrefix + } + + // Initialize stats + stats := &ConfigureStats{} + + // Get service details + serviceDetails := getServiceDetailsFromViperOrCmd(cmd) + exe := api.InitHTTPExecuter(serviceDetails) + + // Phase 1: Configure all artifacts + log.Info().Msg("") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msg("PHASE 1: CONFIGURING ARTIFACTS") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + + deploymentTasks, err := configureAllArtifacts(exe, configData, packageFilter, artifactFilter, + stats, dryRun, batchSize, disableBatch) + if err != nil { + return err + } + + // Phase 2: Deploy artifacts if requested + if len(deploymentTasks) > 0 && !dryRun { + log.Info().Msg("") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msg("PHASE 2: DEPLOYING CONFIGURED ARTIFACTS") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msgf("Deploying %d artifacts with max %d parallel deployments per package", + len(deploymentTasks), parallelDeployments) + + err := deployConfiguredArtifacts(exe, deploymentTasks, deployRetries, deployDelaySeconds, + parallelDeployments, stats) + if err != nil { + log.Error().Msgf("Deployment phase failed: %v", err) + } + } + + // Print summary + printConfigureSummary(stats, dryRun) + + // Return error if there were failures + if stats.ArtifactsFailed > 0 || stats.DeploymentTasksFailed > 0 { + return fmt.Errorf("configuration/deployment completed with errors") + } + + return nil +} + +// ConfigureConfigFile represents a loaded config file with metadata +type ConfigureConfigFile struct { + Config *models.ConfigureConfig + Source string + FileName string +} + +func loadConfigureConfigs(path string) ([]*ConfigureConfigFile, error) { + // Check if path is a file or directory + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("failed to access path: %w", err) + } + + if info.IsDir() { + return loadConfigureConfigsFromFolder(path) + } + return loadConfigureConfigFromFile(path) +} + +func loadConfigureConfigFromFile(path string) ([]*ConfigureConfigFile, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + + var cfg models.ConfigureConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse YAML: %w", err) + } + + return []*ConfigureConfigFile{ + { + Config: &cfg, + Source: path, + FileName: filepath.Base(path), + }, + }, nil +} + +func loadConfigureConfigsFromFolder(folderPath string) ([]*ConfigureConfigFile, error) { + var configFiles []*ConfigureConfigFile + + entries, err := os.ReadDir(folderPath) + if err != nil { + return nil, fmt.Errorf("failed to read directory: %w", err) + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + + // Match YAML files (*.yml, *.yaml) + name := entry.Name() + if !strings.HasSuffix(name, ".yml") && !strings.HasSuffix(name, ".yaml") { + continue + } + + filePath := filepath.Join(folderPath, name) + data, err := os.ReadFile(filePath) + if err != nil { + log.Warn().Msgf("Failed to read config file %s: %v", name, err) + continue + } + + var cfg models.ConfigureConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + log.Warn().Msgf("Failed to parse config file %s: %v", name, err) + continue + } + + configFiles = append(configFiles, &ConfigureConfigFile{ + Config: &cfg, + Source: filePath, + FileName: name, + }) + } + + if len(configFiles) == 0 { + return nil, fmt.Errorf("no valid configuration files found in folder: %s", folderPath) + } + + log.Info().Msgf("Loaded %d configuration file(s) from folder", len(configFiles)) + return configFiles, nil +} + +func mergeConfigureConfigs(configFiles []*ConfigureConfigFile, overridePrefix string) *models.ConfigureConfig { + merged := &models.ConfigureConfig{ + Packages: []models.ConfigurePackage{}, + } + + // Use override prefix if provided, otherwise use first config's prefix + if overridePrefix != "" { + merged.DeploymentPrefix = overridePrefix + } else if len(configFiles) > 0 && configFiles[0].Config.DeploymentPrefix != "" { + merged.DeploymentPrefix = configFiles[0].Config.DeploymentPrefix + } + + // Merge all packages from all config files + for _, configFile := range configFiles { + log.Info().Msgf(" Merging packages from: %s", configFile.FileName) + merged.Packages = append(merged.Packages, configFile.Config.Packages...) + } + + return merged +} + +func configureAllArtifacts(exe *httpclnt.HTTPExecuter, cfg *models.ConfigureConfig, + packageFilter, artifactFilter []string, stats *ConfigureStats, dryRun bool, + batchSize int, disableBatch bool) ([]DeploymentTask, error) { + + var deploymentTasks []DeploymentTask + configuration := api.NewConfiguration(exe) + + for _, pkg := range cfg.Packages { + stats.PackagesProcessed++ + + // Apply deployment prefix to package ID + packageID := pkg.ID + if cfg.DeploymentPrefix != "" { + packageID = cfg.DeploymentPrefix + packageID + } + + // Apply package filter + if len(packageFilter) > 0 && !shouldInclude(pkg.ID, packageFilter) { + log.Info().Msgf("Skipping package %s (filtered out)", packageID) + continue + } + + log.Info().Msg("") + log.Info().Msgf("📦 Processing package: %s", packageID) + if pkg.DisplayName != "" { + log.Info().Msgf(" Display Name: %s", pkg.DisplayName) + } + + packageHasError := false + + for _, artifact := range pkg.Artifacts { + stats.ArtifactsProcessed++ + + // Apply deployment prefix to artifact ID + artifactID := artifact.ID + if cfg.DeploymentPrefix != "" { + artifactID = cfg.DeploymentPrefix + artifactID + } + + // Apply artifact filter + if len(artifactFilter) > 0 && !shouldInclude(artifact.ID, artifactFilter) { + log.Info().Msgf(" Skipping artifact %s (filtered out)", artifactID) + continue + } + + log.Info().Msg("") + log.Info().Msgf(" 🔧 Configuring artifact: %s", artifactID) + if artifact.DisplayName != "" { + log.Info().Msgf(" Display Name: %s", artifact.DisplayName) + } + log.Info().Msgf(" Type: %s", artifact.Type) + log.Info().Msgf(" Version: %s", artifact.Version) + log.Info().Msgf(" Parameters: %d", len(artifact.Parameters)) + + if dryRun { + log.Info().Msg(" [DRY RUN] Would update the following parameters:") + for _, param := range artifact.Parameters { + log.Info().Msgf(" - %s = %s", param.Key, param.Value) + } + stats.ArtifactsConfigured++ + stats.ParametersUpdated += len(artifact.Parameters) + + // Queue for deployment if requested + if artifact.Deploy || pkg.Deploy { + stats.DeploymentTasksQueued++ + log.Info().Msgf(" [DRY RUN] Would deploy after configuration") + } + continue + } + + // Determine batch settings + useBatch := !disableBatch + effectiveBatchSize := batchSize + + if artifact.Batch != nil { + useBatch = artifact.Batch.Enabled && !disableBatch + if artifact.Batch.BatchSize > 0 { + effectiveBatchSize = artifact.Batch.BatchSize + } + } + + // Update configuration parameters + var configErr error + if useBatch && len(artifact.Parameters) > 0 { + configErr = updateParametersBatch(exe, configuration, artifactID, artifact.Version, + artifact.Parameters, effectiveBatchSize, stats) + } else { + configErr = updateParametersIndividual(configuration, artifactID, artifact.Version, + artifact.Parameters, stats) + } + + if configErr != nil { + log.Error().Msgf(" ❌ Failed to configure artifact: %v", configErr) + stats.ArtifactsFailed++ + packageHasError = true + continue + } + + stats.ArtifactsConfigured++ + log.Info().Msgf(" ✅ Successfully configured %d parameters", len(artifact.Parameters)) + + // Queue for deployment if requested + if artifact.Deploy || pkg.Deploy { + deploymentTasks = append(deploymentTasks, DeploymentTask{ + ArtifactID: artifactID, + ArtifactType: artifact.Type, + PackageID: packageID, + DisplayName: artifact.DisplayName, + }) + stats.DeploymentTasksQueued++ + log.Info().Msgf(" 📋 Queued for deployment") + } + } + + if packageHasError { + stats.PackagesWithErrors++ + } + } + + return deploymentTasks, nil +} + +func updateParametersBatch(exe *httpclnt.HTTPExecuter, configuration *api.Configuration, + artifactID, version string, parameters []models.ConfigurationParameter, + batchSize int, stats *ConfigureStats) error { + + log.Info().Msgf(" Using batch operations (batch size: %d)", batchSize) + + // Get current configuration to verify parameters exist + currentConfig, err := configuration.Get(artifactID, version) + if err != nil { + return fmt.Errorf("failed to get current configuration: %w", err) + } + + // Build batch request + batch := exe.NewBatchRequest() + validParams := 0 + + for _, param := range parameters { + // Verify parameter exists + existingParam := api.FindParameterByKey(param.Key, currentConfig.Root.Results) + if existingParam == nil { + log.Warn().Msgf(" ⚠️ Parameter %s not found in artifact, skipping", param.Key) + stats.ParametersFailed++ + continue + } + + // Add to batch + requestBody := fmt.Sprintf(`{"ParameterValue":"%s"}`, escapeJSON(param.Value)) + urlPath := fmt.Sprintf("/api/v1/IntegrationDesigntimeArtifacts(Id='%s',Version='%s')/$links/Configurations('%s')", + artifactID, version, param.Key) + + batch.AddOperation(httpclnt.BatchOperation{ + Method: "PUT", + Path: urlPath, + Body: []byte(requestBody), + ContentID: fmt.Sprintf("param_%d", validParams), + Headers: map[string]string{ + "Content-Type": "application/json", + }, + }) + validParams++ + } + + if validParams == 0 { + return fmt.Errorf("no valid parameters to update") + } + + // Execute batch in chunks + resp, err := batch.ExecuteInBatches(batchSize) + if err != nil { + log.Warn().Msgf(" ⚠️ Batch operation failed: %v, falling back to individual requests", err) + return updateParametersIndividual(configuration, artifactID, version, parameters, stats) + } + + stats.BatchRequestsExecuted++ + + // Process batch results + successCount := 0 + failCount := 0 + + for _, opResp := range resp.Operations { + if opResp.Error != nil { + failCount++ + stats.ParametersFailed++ + } else if opResp.StatusCode >= 200 && opResp.StatusCode < 300 { + successCount++ + stats.ParametersUpdated++ + } else { + failCount++ + stats.ParametersFailed++ + } + } + + if failCount > 0 { + return fmt.Errorf("%d parameters failed to update in batch", failCount) + } + + return nil +} + +func updateParametersIndividual(configuration *api.Configuration, artifactID, version string, + parameters []models.ConfigurationParameter, stats *ConfigureStats) error { + + log.Info().Msgf(" Using individual requests") + + failCount := 0 + successCount := 0 + + for _, param := range parameters { + err := configuration.Update(artifactID, version, param.Key, param.Value) + if err != nil { + log.Error().Msgf(" ❌ Failed to update parameter %s: %v", param.Key, err) + stats.ParametersFailed++ + failCount++ + } else { + stats.ParametersUpdated++ + stats.IndividualRequestsUsed++ + successCount++ + } + } + + if failCount > 0 { + return fmt.Errorf("%d parameters failed to update", failCount) + } + + return nil +} + +func deployConfiguredArtifacts(exe *httpclnt.HTTPExecuter, tasks []DeploymentTask, + deployRetries, deployDelaySeconds, parallelDeployments int, stats *ConfigureStats) error { + + // Group tasks by package + packageTasks := make(map[string][]DeploymentTask) + for _, task := range tasks { + packageTasks[task.PackageID] = append(packageTasks[task.PackageID], task) + } + + log.Info().Msgf("Deploying artifacts across %d packages", len(packageTasks)) + + var wg sync.WaitGroup + resultsChan := make(chan deployResult, len(tasks)) + + // Deploy all artifacts in parallel + for packageID, pkgTasks := range packageTasks { + log.Info().Msgf("Package %s: deploying %d artifacts", packageID, len(pkgTasks)) + + // Process artifacts in this package with controlled parallelism + semaphore := make(chan struct{}, parallelDeployments) + + for _, task := range pkgTasks { + wg.Add(1) + go func(t DeploymentTask) { + defer wg.Done() + semaphore <- struct{}{} // Acquire + defer func() { <-semaphore }() // Release + + log.Info().Msgf(" Deploying %s (type: %s)", t.ArtifactID, t.ArtifactType) + + deployErr := deployArtifact(exe, t, deployRetries, deployDelaySeconds) + resultsChan <- deployResult{Task: t, Error: deployErr} + }(task) + } + } + + // Wait for all deployments + go func() { + wg.Wait() + close(resultsChan) + }() + + // Collect results + for result := range resultsChan { + if result.Error != nil { + log.Error().Msgf(" ❌ Failed to deploy %s: %v", result.Task.ArtifactID, result.Error) + stats.DeploymentTasksFailed++ + } else { + log.Info().Msgf(" ✅ Successfully deployed %s", result.Task.ArtifactID) + stats.DeploymentTasksSuccessful++ + stats.ArtifactsDeployed++ + } + } + + return nil +} + +func deployArtifact(exe *httpclnt.HTTPExecuter, task DeploymentTask, + maxRetries, delaySeconds int) error { + + // Initialize designtime artifact based on type + dt := api.NewDesigntimeArtifact(task.ArtifactType, exe) + + // Initialize runtime artifact for status checking + rt := api.NewRuntime(exe) + + // Deploy the artifact + log.Info().Msgf(" Deploying %s (type: %s)", task.ArtifactID, task.ArtifactType) + err := dt.Deploy(task.ArtifactID) + if err != nil { + return fmt.Errorf("failed to initiate deployment: %w", err) + } + + log.Info().Msgf(" Deployment triggered for %s", task.ArtifactID) + + // Poll for deployment status + for i := 0; i < maxRetries; i++ { + time.Sleep(time.Duration(delaySeconds) * time.Second) + + version, status, err := rt.Get(task.ArtifactID) + if err != nil { + log.Warn().Msgf(" Failed to get deployment status (attempt %d/%d): %v", + i+1, maxRetries, err) + continue + } + + log.Info().Msgf(" Check %d/%d - Status: %s, Version: %s", i+1, maxRetries, status, version) + + if version == "NOT_DEPLOYED" { + continue + } + + if status == "STARTED" { + return nil + } else if status != "STARTING" { + // Get error details + time.Sleep(time.Duration(delaySeconds) * time.Second) + errorMessage, err := rt.GetErrorInfo(task.ArtifactID) + if err != nil { + return fmt.Errorf("deployment failed with status %s: %w", status, err) + } + return fmt.Errorf("deployment failed with status %s: %s", status, errorMessage) + } + } + + return fmt.Errorf("deployment status check timed out after %d attempts", maxRetries) +} + +func printConfigureSummary(stats *ConfigureStats, dryRun bool) { + log.Info().Msg("") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + if dryRun { + log.Info().Msg("DRY RUN SUMMARY") + } else { + log.Info().Msg("CONFIGURATION SUMMARY") + } + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msgf("Packages processed: %d", stats.PackagesProcessed) + log.Info().Msgf("Packages with errors: %d", stats.PackagesWithErrors) + log.Info().Msgf("Artifacts processed: %d", stats.ArtifactsProcessed) + log.Info().Msgf("Artifacts configured: %d", stats.ArtifactsConfigured) + log.Info().Msgf("Artifacts failed: %d", stats.ArtifactsFailed) + log.Info().Msgf("Parameters updated: %d", stats.ParametersUpdated) + log.Info().Msgf("Parameters failed: %d", stats.ParametersFailed) + + if !dryRun { + log.Info().Msg("") + log.Info().Msg("Performance:") + log.Info().Msgf("Batch requests executed: %d", stats.BatchRequestsExecuted) + log.Info().Msgf("Individual requests used: %d", stats.IndividualRequestsUsed) + } + + if stats.DeploymentTasksQueued > 0 { + log.Info().Msg("") + log.Info().Msg("Deployment:") + log.Info().Msgf("Deployment tasks queued: %d", stats.DeploymentTasksQueued) + if !dryRun { + log.Info().Msgf("Deployments successful: %d", stats.DeploymentTasksSuccessful) + log.Info().Msgf("Deployments failed: %d", stats.DeploymentTasksFailed) + log.Info().Msgf("Artifacts deployed: %d", stats.ArtifactsDeployed) + } + } + + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + + if stats.ArtifactsFailed > 0 || stats.DeploymentTasksFailed > 0 { + log.Error().Msg("❌ Configuration/Deployment completed with errors") + } else if dryRun { + log.Info().Msg("✅ Dry run completed successfully") + } else { + log.Info().Msg("✅ Configuration/Deployment completed successfully") + } +} + +func escapeJSON(s string) string { + // Simple JSON string escaping + s = strings.ReplaceAll(s, "\\", "\\\\") + s = strings.ReplaceAll(s, "\"", "\\\"") + s = strings.ReplaceAll(s, "\n", "\\n") + s = strings.ReplaceAll(s, "\r", "\\r") + s = strings.ReplaceAll(s, "\t", "\\t") + return s +} diff --git a/internal/cmd/deploy.go b/internal/cmd/deploy.go index ca80bbe..588bce8 100644 --- a/internal/cmd/deploy.go +++ b/internal/cmd/deploy.go @@ -18,10 +18,14 @@ func NewDeployCommand() *cobra.Command { Use: "deploy", Short: "Deploy designtime artifact to runtime", Long: `Deploy artifact from designtime to -runtime of SAP Integration Suite tenant.`, +runtime of SAP Integration Suite tenant. + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'deploy' section. CLI flags override config file settings.`, PreRunE: func(cmd *cobra.Command, args []string) error { // Validate the artifact type - artifactType := config.GetString(cmd, "artifact-type") + artifactType := config.GetStringWithFallback(cmd, "artifact-type", "deploy.artifactType") switch artifactType { case "MessageMapping", "ScriptCollection", "Integration", "ValueMapping": default: @@ -40,12 +44,13 @@ runtime of SAP Integration Suite tenant.`, } // Define cobra flags, the default value has the lowest (least significant) precedence - deployCmd.Flags().StringSlice("artifact-ids", nil, "Comma separated list of artifact IDs") - deployCmd.Flags().Int("delay-length", 30, "Delay (in seconds) between each check of artifact deployment status") - deployCmd.Flags().Int("max-check-limit", 10, "Max number of times to check for artifact deployment status") + // Note: These can be set in config file under 'deploy' key + deployCmd.Flags().StringSlice("artifact-ids", nil, "Comma separated list of artifact IDs (config: deploy.artifactIds)") + deployCmd.Flags().Int("delay-length", 30, "Delay (in seconds) between each check of artifact deployment status (config: deploy.delayLength)") + deployCmd.Flags().Int("max-check-limit", 10, "Max number of times to check for artifact deployment status (config: deploy.maxCheckLimit)") // To set to false, use --compare-versions=false - deployCmd.Flags().Bool("compare-versions", true, "Perform version comparison of design time against runtime before deployment") - deployCmd.Flags().String("artifact-type", "Integration", "Artifact type. Allowed values: Integration, MessageMapping, ScriptCollection, ValueMapping") + deployCmd.Flags().Bool("compare-versions", true, "Perform version comparison of design time against runtime before deployment (config: deploy.compareVersions)") + deployCmd.Flags().String("artifact-type", "Integration", "Artifact type. Allowed values: Integration, MessageMapping, ScriptCollection, ValueMapping (config: deploy.artifactType)") _ = deployCmd.MarkFlagRequired("artifact-ids") return deployCmd @@ -54,13 +59,14 @@ runtime of SAP Integration Suite tenant.`, func runDeploy(cmd *cobra.Command) error { serviceDetails := api.GetServiceDetails(cmd) - artifactType := config.GetString(cmd, "artifact-type") + // Support reading from config file under 'deploy' key + artifactType := config.GetStringWithFallback(cmd, "artifact-type", "deploy.artifactType") log.Info().Msgf("Executing deploy %v command", artifactType) - artifactIds := config.GetStringSlice(cmd, "artifact-ids") - delayLength := config.GetInt(cmd, "delay-length") - maxCheckLimit := config.GetInt(cmd, "max-check-limit") - compareVersions := config.GetBool(cmd, "compare-versions") + artifactIds := config.GetStringSliceWithFallback(cmd, "artifact-ids", "deploy.artifactIds") + delayLength := config.GetIntWithFallback(cmd, "delay-length", "deploy.delayLength") + maxCheckLimit := config.GetIntWithFallback(cmd, "max-check-limit", "deploy.maxCheckLimit") + compareVersions := config.GetBoolWithFallback(cmd, "compare-versions", "deploy.compareVersions") err := deployArtifacts(artifactIds, artifactType, delayLength, maxCheckLimit, compareVersions, serviceDetails) if err != nil { diff --git a/internal/cmd/flashpipe_orchestrator.go b/internal/cmd/flashpipe_orchestrator.go new file mode 100644 index 0000000..6eb0a98 --- /dev/null +++ b/internal/cmd/flashpipe_orchestrator.go @@ -0,0 +1,974 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/engswee/flashpipe/internal/api" + "github.com/engswee/flashpipe/internal/config" + "github.com/engswee/flashpipe/internal/deploy" + "github.com/engswee/flashpipe/internal/models" + flashpipeSync "github.com/engswee/flashpipe/internal/sync" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// OperationMode defines the orchestrator operation mode +type OperationMode string + +const ( + ModeUpdateAndDeploy OperationMode = "update-and-deploy" + ModeUpdateOnly OperationMode = "update-only" + ModeDeployOnly OperationMode = "deploy-only" +) + +// ProcessingStats tracks processing statistics +type ProcessingStats struct { + PackagesUpdated int + PackagesDeployed int + PackagesFailed int + PackagesFiltered int + ArtifactsTotal int + ArtifactsDeployedSuccess int + ArtifactsDeployedFailed int + ArtifactsFiltered int + UpdateFailures int + DeployFailures int + SuccessfulPackageUpdates map[string]bool + SuccessfulArtifactUpdates map[string]bool + SuccessfulArtifactDeploys map[string]bool + FailedPackageUpdates map[string]bool + FailedArtifactUpdates map[string]bool + FailedArtifactDeploys map[string]bool +} + +// DeploymentTask represents an artifact ready for deployment +type DeploymentTask struct { + ArtifactID string + ArtifactType string + PackageID string + DisplayName string +} + +func NewFlashpipeOrchestratorCommand() *cobra.Command { + var ( + packagesDir string + deployConfig string + deploymentPrefix string + packageFilter string + artifactFilter string + keepTemp bool + debugMode bool + configPattern string + mergeConfigs bool + updateMode bool + updateOnlyMode bool + deployOnlyMode bool + deployRetries int + deployDelaySeconds int + parallelDeployments int + ) + + orchestratorCmd := &cobra.Command{ + Use: "orchestrator", + Short: "Orchestrate SAP CPI artifact updates and deployments", + SilenceUsage: true, // Don't show usage on execution errors + Long: `Orchestrate the complete deployment lifecycle for SAP CPI artifacts. + +This command handles: + - Updates artifacts in SAP CPI tenant with modified MANIFEST.MF and parameters + - Deploys artifacts to make them active (in parallel for faster execution) + - Supports deployment prefixes for multi-environment scenarios + - Intelligent artifact grouping by type for efficient deployment + - Filter by specific packages or artifacts + - Load configs from files, folders, or remote URLs + - Configure via YAML file for repeatable deployments + +Configuration Sources: + The --deploy-config flag accepts: + - Single file: ./001-deploy-config.yml + - Folder: ./configs (processes all matching files alphabetically) + - Remote URL: https://raw.githubusercontent.com/org/repo/main/config.yml + + Use --orchestrator-config to load all settings from a YAML file: + - Sets all flags from YAML + - CLI flags override YAML settings + +Operation Modes: + --update Update and deploy artifacts (default) + --update-only Only update artifacts, don't deploy + --deploy-only Only deploy artifacts, don't update + +Deployment Strategy: + 1. Update Phase: All packages and artifacts are updated first + 2. Deploy Phase: All artifacts are deployed in parallel + - Deployments are triggered concurrently per package + - Status is polled for all deployments simultaneously + - Configurable parallelism and retry settings + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'orchestrator' section. CLI flags override config file settings.`, + Example: ` # Update and deploy with config from global flashpipe.yaml + flashpipe orchestrator --update + + # Load specific config file + flashpipe orchestrator --config ./my-config.yml --update + + # Override settings via CLI flags + flashpipe orchestrator --config ./my-config.yml \ + --deployment-prefix DEV --parallel-deployments 5`, + RunE: func(cmd *cobra.Command, args []string) error { + // Determine operation mode + mode := ModeUpdateAndDeploy + if updateOnlyMode { + mode = ModeUpdateOnly + } else if deployOnlyMode { + mode = ModeDeployOnly + } + + // Load from viper config if available (CLI flags override config file) + if !cmd.Flags().Changed("packages-dir") && viper.IsSet("orchestrator.packagesDir") { + packagesDir = viper.GetString("orchestrator.packagesDir") + } + if !cmd.Flags().Changed("deploy-config") && viper.IsSet("orchestrator.deployConfig") { + deployConfig = viper.GetString("orchestrator.deployConfig") + } + if !cmd.Flags().Changed("deployment-prefix") && viper.IsSet("orchestrator.deploymentPrefix") { + deploymentPrefix = viper.GetString("orchestrator.deploymentPrefix") + } + if !cmd.Flags().Changed("package-filter") && viper.IsSet("orchestrator.packageFilter") { + packageFilter = viper.GetString("orchestrator.packageFilter") + } + if !cmd.Flags().Changed("artifact-filter") && viper.IsSet("orchestrator.artifactFilter") { + artifactFilter = viper.GetString("orchestrator.artifactFilter") + } + if !cmd.Flags().Changed("config-pattern") && viper.IsSet("orchestrator.configPattern") { + configPattern = viper.GetString("orchestrator.configPattern") + } + if !cmd.Flags().Changed("merge-configs") && viper.IsSet("orchestrator.mergeConfigs") { + mergeConfigs = viper.GetBool("orchestrator.mergeConfigs") + } + if !cmd.Flags().Changed("keep-temp") && viper.IsSet("orchestrator.keepTemp") { + keepTemp = viper.GetBool("orchestrator.keepTemp") + } + if !updateMode && !updateOnlyMode && !deployOnlyMode && viper.IsSet("orchestrator.mode") { + switch viper.GetString("orchestrator.mode") { + case "update-and-deploy": + mode = ModeUpdateAndDeploy + case "update-only": + mode = ModeUpdateOnly + case "deploy-only": + mode = ModeDeployOnly + } + } + if !cmd.Flags().Changed("deploy-retries") && viper.IsSet("orchestrator.deployRetries") { + deployRetries = viper.GetInt("orchestrator.deployRetries") + } + if !cmd.Flags().Changed("deploy-delay") && viper.IsSet("orchestrator.deployDelaySeconds") { + deployDelaySeconds = viper.GetInt("orchestrator.deployDelaySeconds") + } + if !cmd.Flags().Changed("parallel-deployments") && viper.IsSet("orchestrator.parallelDeployments") { + parallelDeployments = viper.GetInt("orchestrator.parallelDeployments") + } + + // Validate required parameters + if deployConfig == "" { + return fmt.Errorf("--deploy-config is required (set via CLI flag or in config file under 'orchestrator.deployConfig')") + } + + // Set defaults for deployment settings + if deployRetries == 0 { + deployRetries = 5 + } + if deployDelaySeconds == 0 { + deployDelaySeconds = 15 + } + if parallelDeployments == 0 { + parallelDeployments = 3 + } + + return runOrchestrator(cmd, mode, packagesDir, deployConfig, + deploymentPrefix, packageFilter, artifactFilter, keepTemp, debugMode, + configPattern, mergeConfigs, deployRetries, deployDelaySeconds, parallelDeployments) + }, + } + + // Flags + orchestratorCmd.Flags().StringVarP(&packagesDir, "packages-dir", "d", "", "Directory containing packages (config: orchestrator.packagesDir)") + orchestratorCmd.Flags().StringVarP(&deployConfig, "deploy-config", "c", "", "Path to deployment config file/folder/URL (config: orchestrator.deployConfig)") + orchestratorCmd.Flags().StringVarP(&deploymentPrefix, "deployment-prefix", "p", "", "Deployment prefix for package/artifact IDs (config: orchestrator.deploymentPrefix)") + orchestratorCmd.Flags().StringVar(&packageFilter, "package-filter", "", "Comma-separated list of packages to include (config: orchestrator.packageFilter)") + orchestratorCmd.Flags().StringVar(&artifactFilter, "artifact-filter", "", "Comma-separated list of artifacts to include (config: orchestrator.artifactFilter)") + orchestratorCmd.Flags().BoolVar(&keepTemp, "keep-temp", false, "Keep temporary directory after execution (config: orchestrator.keepTemp)") + orchestratorCmd.Flags().BoolVar(&debugMode, "debug", false, "Enable debug logging") + orchestratorCmd.Flags().StringVar(&configPattern, "config-pattern", "*.y*ml", "File pattern for config files in folders (config: orchestrator.configPattern)") + orchestratorCmd.Flags().BoolVar(&mergeConfigs, "merge-configs", false, "Merge multiple configs into single deployment (config: orchestrator.mergeConfigs)") + orchestratorCmd.Flags().BoolVar(&updateMode, "update", false, "Update and deploy artifacts") + orchestratorCmd.Flags().BoolVar(&updateOnlyMode, "update-only", false, "Only update artifacts, don't deploy") + orchestratorCmd.Flags().BoolVar(&deployOnlyMode, "deploy-only", false, "Only deploy artifacts, don't update") + orchestratorCmd.Flags().IntVar(&deployRetries, "deploy-retries", 0, "Number of retries for deployment status checks (config: orchestrator.deployRetries, default: 5)") + orchestratorCmd.Flags().IntVar(&deployDelaySeconds, "deploy-delay", 0, "Delay in seconds between deployment status checks (config: orchestrator.deployDelaySeconds, default: 15)") + orchestratorCmd.Flags().IntVar(¶llelDeployments, "parallel-deployments", 0, "Number of parallel deployments per package (config: orchestrator.parallelDeployments, default: 3)") + + return orchestratorCmd +} + +// getServiceDetailsFromViperOrCmd reads service credentials from viper config or CLI flags +// This allows the orchestrator to use credentials from the global config file +func getServiceDetailsFromViperOrCmd(cmd *cobra.Command) *api.ServiceDetails { + // Try to read from CLI flags first (via api.GetServiceDetails) + serviceDetails := api.GetServiceDetails(cmd) + + // If host is empty, credentials weren't provided via CLI flags + // Try to read from viper (global config file) + if serviceDetails.Host == "" { + tmnHost := viper.GetString("tmn-host") + oauthHost := viper.GetString("oauth-host") + + if tmnHost == "" { + log.Debug().Msg("No CPI credentials found in CLI flags or config file") + return nil // No credentials found + } + + log.Debug().Msg("Using CPI credentials from config file (viper)") + log.Debug().Msgf(" tmn-host: %s", tmnHost) + + // Use OAuth if oauth-host is set + if oauthHost != "" { + log.Debug().Msgf(" oauth-host: %s", oauthHost) + + oauthPath := viper.GetString("oauth-path") + if oauthPath == "" { + oauthPath = "/oauth/token" // Default value + } + + return &api.ServiceDetails{ + Host: tmnHost, + OauthHost: oauthHost, + OauthClientId: viper.GetString("oauth-clientid"), + OauthClientSecret: viper.GetString("oauth-clientsecret"), + OauthPath: oauthPath, + } + } else { + log.Debug().Msg(" Using Basic Auth") + return &api.ServiceDetails{ + Host: tmnHost, + Userid: viper.GetString("tmn-userid"), + Password: viper.GetString("tmn-password"), + } + } + } + + log.Debug().Msg("Using CPI credentials from CLI flags") + return serviceDetails +} + +func runOrchestrator(cmd *cobra.Command, mode OperationMode, packagesDir, deployConfigPath, + deploymentPrefix, packageFilterStr, artifactFilterStr string, keepTemp, debugMode bool, + configPattern string, mergeConfigs bool, deployRetries, deployDelaySeconds, parallelDeployments int) error { + + log.Info().Msg("Starting flashpipe orchestrator") + log.Info().Msgf("Deployment Strategy: Two-phase with parallel deployment") + log.Info().Msgf(" Phase 1: Update all artifacts") + log.Info().Msgf(" Phase 2: Deploy all artifacts in parallel (max %d concurrent)", parallelDeployments) + + // Validate deployment prefix + if err := deploy.ValidateDeploymentPrefix(deploymentPrefix); err != nil { + return err + } + + // Parse filters + packageFilter := parseFilter(packageFilterStr) + artifactFilter := parseFilter(artifactFilterStr) + + // Initialize stats + stats := ProcessingStats{ + SuccessfulArtifactUpdates: make(map[string]bool), + SuccessfulPackageUpdates: make(map[string]bool), + SuccessfulArtifactDeploys: make(map[string]bool), + FailedArtifactUpdates: make(map[string]bool), + FailedPackageUpdates: make(map[string]bool), + FailedArtifactDeploys: make(map[string]bool), + } + + // Setup config loader + configLoader := deploy.NewConfigLoader() + configLoader.Debug = debugMode + configLoader.FilePattern = configPattern + + // Get auth settings from viper/config for remote URLs + if viper.IsSet("host") { + // Use CPI credentials from global config if deploying from URL + configLoader.Username = config.GetString(cmd, "username") + configLoader.Password = config.GetString(cmd, "password") + } + + if err := configLoader.DetectSource(deployConfigPath); err != nil { + return fmt.Errorf("failed to detect config source: %w", err) + } + + log.Info().Msgf("Loading config from: %s (type: %s)", deployConfigPath, configLoader.Source) + configFiles, err := configLoader.LoadConfigs() + if err != nil { + return fmt.Errorf("failed to load deployment config: %w", err) + } + + log.Info().Msgf("Loaded %d config file(s)", len(configFiles)) + + // Create temporary work directory if needed + var workDir string + if mode != ModeDeployOnly { + tempDir, err := os.MkdirTemp("", "flashpipe-orchestrator-*") + if err != nil { + return fmt.Errorf("failed to create temp directory: %w", err) + } + workDir = tempDir + + if !keepTemp { + defer os.RemoveAll(tempDir) + } else { + log.Info().Msgf("Temporary directory: %s", tempDir) + } + } + + log.Info().Msgf("Mode: %s", mode) + log.Info().Msgf("Packages Directory: %s", packagesDir) + + if len(packageFilter) > 0 { + log.Info().Msgf("Package filter: %s", strings.Join(packageFilter, ", ")) + } + if len(artifactFilter) > 0 { + log.Info().Msgf("Artifact filter: %s", strings.Join(artifactFilter, ", ")) + } + + // Get service details once (shared across all operations) + // Read credentials from viper if not provided via CLI flags + serviceDetails := getServiceDetailsFromViperOrCmd(cmd) + if serviceDetails == nil { + return fmt.Errorf("missing CPI credentials: provide via --config file or CLI flags (--tmn-host, --oauth-host, etc.)") + } + + // Validate serviceDetails has required fields + if serviceDetails.Host == "" { + return fmt.Errorf("CPI host (tmn-host) is required but not provided") + } + + log.Debug().Msg("CPI credentials successfully loaded:") + log.Debug().Msgf(" Host: %s", serviceDetails.Host) + if serviceDetails.OauthHost != "" { + log.Debug().Msgf(" OAuth Host: %s", serviceDetails.OauthHost) + log.Debug().Msg(" Auth Method: OAuth") + } else { + log.Debug().Msg(" Auth Method: Basic Auth") + } + + // Collect all deployment tasks (will be executed in phase 2) + var deploymentTasks []DeploymentTask + + // Process configs + if mergeConfigs && len(configFiles) > 1 { + log.Info().Msg("Merging multiple configs into single deployment") + + if deploymentPrefix != "" { + log.Warn().Msg("Note: --deployment-prefix is ignored when merging configs with their own prefixes") + } + + mergedConfig, err := deploy.MergeConfigs(configFiles) + if err != nil { + return fmt.Errorf("failed to merge configs: %w", err) + } + + tasks, err := processPackages(mergedConfig, false, mode, packagesDir, workDir, + packageFilter, artifactFilter, &stats, serviceDetails) + if err != nil { + return err + } + deploymentTasks = append(deploymentTasks, tasks...) + } else { + for _, configFile := range configFiles { + if len(configFiles) > 1 { + log.Info().Msgf("Processing Config: %s", configFile.FileName) + } + + // Override deployment prefix if specified via CLI + if deploymentPrefix != "" { + configFile.Config.DeploymentPrefix = deploymentPrefix + } + + log.Info().Msgf("Deployment Prefix: %s", configFile.Config.DeploymentPrefix) + + tasks, err := processPackages(configFile.Config, true, mode, packagesDir, workDir, + packageFilter, artifactFilter, &stats, serviceDetails) + if err != nil { + log.Error().Msgf("Failed to process config %s: %v", configFile.FileName, err) + continue + } + deploymentTasks = append(deploymentTasks, tasks...) + } + } + + // Phase 2: Deploy all artifacts in parallel (if not update-only mode) + if mode != ModeUpdateOnly && len(deploymentTasks) > 0 { + log.Info().Msg("") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msg("PHASE 2: DEPLOYING ALL ARTIFACTS IN PARALLEL") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msgf("Total artifacts to deploy: %d", len(deploymentTasks)) + log.Info().Msgf("Max concurrent deployments: %d", parallelDeployments) + log.Info().Msg("") + + err := deployAllArtifactsParallel(deploymentTasks, parallelDeployments, deployRetries, + deployDelaySeconds, &stats, serviceDetails) + if err != nil { + log.Error().Msgf("Deployment phase failed: %v", err) + } + } + + // Print summary + printSummary(&stats) + + // Return error if there were failures + if stats.PackagesFailed > 0 || stats.UpdateFailures > 0 || stats.DeployFailures > 0 { + return fmt.Errorf("deployment completed with failures") + } + + return nil +} + +func processPackages(config *models.DeployConfig, applyPrefix bool, mode OperationMode, + packagesDir, workDir string, packageFilter, artifactFilter []string, + stats *ProcessingStats, serviceDetails *api.ServiceDetails) ([]DeploymentTask, error) { + + var deploymentTasks []DeploymentTask + + // Phase 1: Update all packages and artifacts + if mode != ModeDeployOnly { + log.Info().Msg("") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msg("PHASE 1: UPDATING ALL PACKAGES AND ARTIFACTS") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msg("") + } + + for _, pkg := range config.Packages { + // Apply package filter + if !shouldInclude(pkg.ID, packageFilter) { + log.Debug().Msgf("Skipping package %s (filtered)", pkg.ID) + stats.PackagesFiltered++ + continue + } + + if !pkg.Sync && !pkg.Deploy { + log.Info().Msgf("Skipping package %s (sync=false, deploy=false)", pkg.ID) + continue + } + + log.Info().Msgf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + log.Info().Msgf("📦 Package: %s", pkg.ID) + + packageDir := filepath.Join(packagesDir, pkg.PackageDir) + if !deploy.DirExists(packageDir) { + log.Warn().Msgf("Package directory not found: %s", packageDir) + continue + } + + // Calculate final package ID and name + finalPackageID := pkg.ID + finalPackageName := pkg.DisplayName + if finalPackageName == "" { + finalPackageName = pkg.ID + } + + // Apply prefix if needed + if applyPrefix && config.DeploymentPrefix != "" { + finalPackageID = config.DeploymentPrefix + "" + pkg.ID + finalPackageName = config.DeploymentPrefix + " - " + finalPackageName + } + + log.Info().Msgf("Package ID: %s", finalPackageID) + log.Info().Msgf("Package Name: %s", finalPackageName) + + // Update package metadata + if mode != ModeDeployOnly { + err := updatePackage(&pkg, finalPackageID, finalPackageName, workDir, serviceDetails) + if err != nil { + log.Error().Msgf("Failed to update package %s: %v", pkg.ID, err) + stats.FailedPackageUpdates[pkg.ID] = true + stats.PackagesFailed++ + continue + } + stats.SuccessfulPackageUpdates[pkg.ID] = true + stats.PackagesUpdated++ + } + + // Process artifacts for update + if pkg.Sync && mode != ModeDeployOnly { + if err := updateArtifacts(&pkg, packageDir, finalPackageID, finalPackageName, + config.DeploymentPrefix, workDir, artifactFilter, stats, serviceDetails); err != nil { + log.Error().Msgf("Failed to update artifacts for package %s: %v", pkg.ID, err) + stats.UpdateFailures++ + } + } + + // Collect deployment tasks (will be executed in phase 2) + if pkg.Deploy && mode != ModeUpdateOnly { + tasks := collectDeploymentTasks(&pkg, finalPackageID, config.DeploymentPrefix, + artifactFilter, stats) + deploymentTasks = append(deploymentTasks, tasks...) + } + } + + return deploymentTasks, nil +} + +func updatePackage(pkg *models.Package, finalPackageID, finalPackageName, workDir string, + serviceDetails *api.ServiceDetails) error { + + if serviceDetails == nil { + return fmt.Errorf("serviceDetails is nil - cannot update package") + } + + log.Info().Msg("Updating package in tenant...") + + description := pkg.Description + if description == "" { + description = finalPackageName + } + + shortText := pkg.ShortText + if shortText == "" { + shortText = finalPackageName + } + + // Create package JSON + packageJSON := map[string]interface{}{ + "d": map[string]interface{}{ + "Id": finalPackageID, + "Name": finalPackageName, + "Description": description, + "ShortText": shortText, + }, + } + + jsonData, err := json.MarshalIndent(packageJSON, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal package JSON: %w", err) + } + + // Write to temporary file + packageJSONPath := filepath.Join(workDir, "modified", fmt.Sprintf("package_%s.json", pkg.ID)) + if err := os.MkdirAll(filepath.Dir(packageJSONPath), 0755); err != nil { + return fmt.Errorf("failed to create package JSON directory: %w", err) + } + + if err := os.WriteFile(packageJSONPath, jsonData, 0644); err != nil { + return fmt.Errorf("failed to write package JSON: %w", err) + } + + // Use internal sync package update function + exe := api.InitHTTPExecuter(serviceDetails) + packageSynchroniser := flashpipeSync.NewSyncer("tenant", "CPIPackage", exe) + + err = packageSynchroniser.Exec(flashpipeSync.Request{PackageFile: packageJSONPath}) + if err != nil { + log.Warn().Msgf("Package update warning (may not exist yet): %v", err) + // Don't return error - package might not exist yet + return nil + } + + log.Info().Msg(" ✓ Package metadata updated") + return nil +} + +func updateArtifacts(pkg *models.Package, packageDir, finalPackageID, finalPackageName, prefix, workDir string, + artifactFilter []string, stats *ProcessingStats, serviceDetails *api.ServiceDetails) error { + + updatedCount := 0 + log.Info().Msg("Updating artifacts...") + + if serviceDetails == nil { + return fmt.Errorf("serviceDetails is nil - cannot initialize HTTP executer") + } + if serviceDetails.Host == "" { + return fmt.Errorf("serviceDetails.Host is empty - check CPI credentials in config file") + } + + log.Info().Msgf("DEBUG: ServiceDetails before InitHTTPExecuter:") + log.Info().Msgf(" Host: %s", serviceDetails.Host) + log.Info().Msgf(" OauthHost: %s", serviceDetails.OauthHost) + log.Info().Msgf(" OauthClientId: %s", serviceDetails.OauthClientId) + log.Info().Msgf(" OauthPath: %s", serviceDetails.OauthPath) + log.Info().Msgf(" Userid: %s", serviceDetails.Userid) + + log.Debug().Msgf("Initializing HTTP executer with host: %s", serviceDetails.Host) + exe := api.InitHTTPExecuter(serviceDetails) + if exe == nil { + return fmt.Errorf("failed to initialize HTTP executer") + } + + log.Info().Msgf("DEBUG: exe after InitHTTPExecuter is NOT nil") + + synchroniser := flashpipeSync.New(exe) + if synchroniser == nil { + return fmt.Errorf("failed to initialize synchroniser") + } + + log.Info().Msgf("DEBUG: synchroniser created successfully") + + for _, artifact := range pkg.Artifacts { + // Apply artifact filter + if !shouldInclude(artifact.Id, artifactFilter) { + log.Debug().Msgf("Skipping artifact %s (filtered)", artifact.Id) + stats.ArtifactsFiltered++ + continue + } + + if !artifact.Sync { + log.Debug().Msgf("Skipping artifact %s (sync=false)", artifact.DisplayName) + continue + } + + stats.ArtifactsTotal++ + + artifactDir := filepath.Join(packageDir, artifact.ArtifactDir) + if !deploy.DirExists(artifactDir) { + log.Warn().Msgf("Artifact directory not found: %s", artifactDir) + continue + } + + // Calculate final artifact ID and name + finalArtifactID := artifact.Id + finalArtifactName := artifact.DisplayName + if finalArtifactName == "" { + finalArtifactName = artifact.Id + } + + if prefix != "" { + finalArtifactID = prefix + "_" + artifact.Id + } + + log.Info().Msgf(" Updating: %s", finalArtifactID) + + // Map artifact type for synchroniser (uses simple type names) + artifactType := mapArtifactTypeForSync(artifact.Type) + + // Create temp directory for this artifact + tempArtifactDir := filepath.Join(workDir, artifact.Id) + if err := deploy.CopyDir(artifactDir, tempArtifactDir); err != nil { + log.Error().Msgf("Failed to copy artifact to temp: %v", err) + stats.FailedArtifactUpdates[artifact.Id] = true + continue + } + + // Update MANIFEST.MF + manifestPath := filepath.Join(tempArtifactDir, "META-INF", "MANIFEST.MF") + modifiedManifestPath := filepath.Join(workDir, "modified", artifact.Id, "META-INF", "MANIFEST.MF") + + if deploy.FileExists(manifestPath) { + if err := deploy.UpdateManifestBundleName(manifestPath, finalArtifactID, finalArtifactName, modifiedManifestPath); err != nil { + log.Warn().Msgf("Failed to update MANIFEST.MF: %v", err) + } + } + + // Handle parameters.prop + var modifiedParamsPath string + paramsPath := deploy.FindParametersFile(tempArtifactDir) + + if paramsPath != "" && deploy.FileExists(paramsPath) { + modifiedParamsPath = filepath.Join(workDir, "modified", artifact.Id, "parameters.prop") + + if len(artifact.ConfigOverrides) > 0 { + if err := deploy.MergeParametersFile(paramsPath, artifact.ConfigOverrides, modifiedParamsPath); err != nil { + log.Warn().Msgf("Failed to merge parameters: %v", err) + } else { + log.Debug().Msgf("Applied %d config overrides", len(artifact.ConfigOverrides)) + } + } else { + // No overrides, copy to modified location + data, err := os.ReadFile(paramsPath) + if err == nil { + os.MkdirAll(filepath.Dir(modifiedParamsPath), 0755) + os.WriteFile(modifiedParamsPath, data, 0644) + } + } + } + + // Copy modified manifest to temp artifact dir for sync + if deploy.FileExists(modifiedManifestPath) { + targetManifestPath := filepath.Join(tempArtifactDir, "META-INF", "MANIFEST.MF") + data, err := os.ReadFile(modifiedManifestPath) + if err == nil { + os.WriteFile(targetManifestPath, data, 0644) + } + } + + // Copy modified parameters if exists + if modifiedParamsPath != "" && deploy.FileExists(modifiedParamsPath) { + // Find the actual parameters location in the artifact + actualParamsPath := deploy.FindParametersFile(tempArtifactDir) + data, err := os.ReadFile(modifiedParamsPath) + if err == nil { + os.WriteFile(actualParamsPath, data, 0644) + } + } + + // Call internal sync function + log.Debug().Msgf("DEBUG: About to call SingleArtifactToTenant for %s", finalArtifactID) + log.Debug().Msgf(" synchroniser: %v", synchroniser) + log.Debug().Msgf(" finalPackageID: %s", finalPackageID) + log.Debug().Msgf(" artifactType: %s", artifactType) + + err := synchroniser.SingleArtifactToTenant(finalArtifactID, finalArtifactName, artifactType, + finalPackageID, tempArtifactDir, workDir, "", nil) + + if err != nil { + log.Error().Msgf("Update failed for %s: %v", finalArtifactName, err) + stats.UpdateFailures++ + stats.FailedArtifactUpdates[artifact.Id] = true + continue + } + + log.Info().Msg(" ✓ Updated successfully") + updatedCount++ + stats.SuccessfulArtifactUpdates[finalArtifactID] = true + } + + if updatedCount > 0 { + log.Info().Msgf("✓ Updated %d artifact(s) in package", updatedCount) + } + + return nil +} + +func collectDeploymentTasks(pkg *models.Package, finalPackageID, prefix string, + artifactFilter []string, stats *ProcessingStats) []DeploymentTask { + + var tasks []DeploymentTask + + for _, artifact := range pkg.Artifacts { + // Skip if update failed + if stats.FailedArtifactUpdates[artifact.Id] { + log.Debug().Msgf("Skipping artifact %s (due to failed update)", artifact.Id) + continue + } + + // Apply artifact filter + if !shouldInclude(artifact.Id, artifactFilter) { + log.Debug().Msgf("Skipping artifact %s (filtered)", artifact.Id) + continue + } + + if !artifact.Deploy { + log.Debug().Msgf("Skipping artifact %s (deploy=false)", artifact.DisplayName) + continue + } + + finalArtifactID := artifact.Id + if prefix != "" { + finalArtifactID = prefix + "_" + artifact.Id + } + + artifactType := artifact.Type + if artifactType == "" { + artifactType = "IntegrationFlow" + } + + tasks = append(tasks, DeploymentTask{ + ArtifactID: finalArtifactID, + ArtifactType: artifactType, + PackageID: finalPackageID, + DisplayName: artifact.DisplayName, + }) + } + + return tasks +} + +func deployAllArtifactsParallel(tasks []DeploymentTask, maxConcurrent int, + retries int, delaySeconds int, stats *ProcessingStats, serviceDetails *api.ServiceDetails) error { + + // Group tasks by package for better control + tasksByPackage := make(map[string][]DeploymentTask) + for _, task := range tasks { + tasksByPackage[task.PackageID] = append(tasksByPackage[task.PackageID], task) + } + + // Process each package's deployments + for packageID, packageTasks := range tasksByPackage { + log.Info().Msgf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + log.Info().Msgf("📦 Deploying %d artifacts for package: %s", len(packageTasks), packageID) + + // Deploy artifacts in parallel with semaphore + var wg sync.WaitGroup + semaphore := make(chan struct{}, maxConcurrent) + resultChan := make(chan deployResult, len(packageTasks)) + + for _, task := range packageTasks { + wg.Add(1) + go func(t DeploymentTask) { + defer wg.Done() + + // Acquire semaphore + semaphore <- struct{}{} + defer func() { <-semaphore }() + + // Deploy artifact + // Use mapArtifactTypeForSync because deployArtifacts calls api.NewDesigntimeArtifact + flashpipeType := mapArtifactTypeForSync(t.ArtifactType) + log.Info().Msgf(" → Deploying: %s (type: %s)", t.ArtifactID, t.ArtifactType) + + err := deployArtifacts([]string{t.ArtifactID}, flashpipeType, retries, delaySeconds, true, serviceDetails) + + resultChan <- deployResult{ + Task: t, + Error: err, + } + }(task) + } + + // Wait for all deployments to complete + wg.Wait() + close(resultChan) + + // Process results + successCount := 0 + failureCount := 0 + + for result := range resultChan { + if result.Error != nil { + log.Error().Msgf(" ✗ Deploy failed: %s - %v", result.Task.ArtifactID, result.Error) + stats.ArtifactsDeployedFailed++ + stats.DeployFailures++ + stats.FailedArtifactDeploys[result.Task.ArtifactID] = true + failureCount++ + } else { + log.Info().Msgf(" ✓ Deployed: %s", result.Task.ArtifactID) + stats.ArtifactsDeployedSuccess++ + stats.SuccessfulArtifactDeploys[result.Task.ArtifactID] = true + successCount++ + } + } + + if failureCount == 0 { + log.Info().Msgf("✓ All %d artifacts deployed successfully for package %s", successCount, packageID) + stats.PackagesDeployed++ + } else { + log.Warn().Msgf("⚠ Package %s: %d succeeded, %d failed", packageID, successCount, failureCount) + stats.PackagesFailed++ + } + } + + return nil +} + +type deployResult struct { + Task DeploymentTask + Error error +} + +// mapArtifactType maps artifact types for deployment API calls +func mapArtifactType(artifactType string) string { + switch strings.ToLower(artifactType) { + case "integrationflow", "integration flow", "iflow": + return "IntegrationDesigntimeArtifact" + case "valuemapping", "value mapping": + return "ValueMappingDesigntimeArtifact" + case "messageMapping", "message mapping": + return "MessageMappingDesigntimeArtifact" + case "scriptcollection", "script collection": + return "ScriptCollection" + default: + // Default to integration flow + return "IntegrationDesigntimeArtifact" + } +} + +// mapArtifactTypeForSync maps artifact types for synchroniser (NewDesigntimeArtifact) +func mapArtifactTypeForSync(artifactType string) string { + switch strings.ToLower(artifactType) { + case "integrationflow", "integration flow", "iflow": + return "Integration" + case "valuemapping", "value mapping": + return "ValueMapping" + case "messagemapping", "message mapping": + return "MessageMapping" + case "scriptcollection", "script collection": + return "ScriptCollection" + default: + // Default to integration flow + return "Integration" + } +} + +func parseFilter(filterStr string) []string { + if filterStr == "" { + return nil + } + parts := strings.Split(filterStr, ",") + var result []string + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result +} + +func shouldInclude(id string, filter []string) bool { + if len(filter) == 0 { + return true + } + for _, f := range filter { + if f == id { + return true + } + } + return false +} + +func printSummary(stats *ProcessingStats) { + log.Info().Msg("") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msg("📊 DEPLOYMENT SUMMARY") + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") + log.Info().Msgf("Packages Updated: %d", stats.PackagesUpdated) + log.Info().Msgf("Packages Deployed: %d", stats.PackagesDeployed) + log.Info().Msgf("Packages Failed: %d", stats.PackagesFailed) + log.Info().Msgf("Packages Filtered: %d", stats.PackagesFiltered) + log.Info().Msg("───────────────────────────────────────────────────────────────────────") + log.Info().Msgf("Artifacts Total: %d", stats.ArtifactsTotal) + log.Info().Msgf("Artifacts Updated: %d", len(stats.SuccessfulArtifactUpdates)) + log.Info().Msgf("Artifacts Deployed OK: %d", stats.ArtifactsDeployedSuccess) + log.Info().Msgf("Artifacts Deployed Fail: %d", stats.ArtifactsDeployedFailed) + log.Info().Msgf("Artifacts Filtered: %d", stats.ArtifactsFiltered) + log.Info().Msg("───────────────────────────────────────────────────────────────────────") + + if stats.UpdateFailures > 0 { + log.Warn().Msgf("⚠ Update Failures: %d", stats.UpdateFailures) + log.Info().Msg("Failed Artifact Updates:") + for artifactID := range stats.FailedArtifactUpdates { + log.Info().Msgf(" - %s", artifactID) + } + } + + if stats.DeployFailures > 0 { + log.Warn().Msgf("⚠ Deploy Failures: %d", stats.DeployFailures) + log.Info().Msg("Failed Artifact Deployments:") + for artifactID := range stats.FailedArtifactDeploys { + log.Info().Msgf(" - %s", artifactID) + } + } + + if stats.UpdateFailures == 0 && stats.DeployFailures == 0 { + log.Info().Msg("✓ All operations completed successfully!") + } + + log.Info().Msg("═══════════════════════════════════════════════════════════════════════") +} diff --git a/internal/cmd/package.go b/internal/cmd/package.go index d35653b..6340261 100644 --- a/internal/cmd/package.go +++ b/internal/cmd/package.go @@ -1,13 +1,14 @@ package cmd import ( + "time" + "github.com/engswee/flashpipe/internal/analytics" "github.com/engswee/flashpipe/internal/api" "github.com/engswee/flashpipe/internal/config" "github.com/engswee/flashpipe/internal/sync" "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "time" ) func NewPackageCommand() *cobra.Command { @@ -17,7 +18,11 @@ func NewPackageCommand() *cobra.Command { Aliases: []string{"pkg"}, Short: "Create/update integration package", Long: `Create or update integration package on the -SAP Integration Suite tenant.`, +SAP Integration Suite tenant. + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'update.package' section. CLI flags override config file settings.`, RunE: func(cmd *cobra.Command, args []string) (err error) { startTime := time.Now() if err = runUpdatePackage(cmd); err != nil { @@ -29,7 +34,8 @@ SAP Integration Suite tenant.`, } // Define cobra flags, the default value has the lowest (least significant) precedence - packageCmd.Flags().String("package-file", "", "Path to location of package file") + // Note: These can be set in config file under 'update.package' key + packageCmd.Flags().String("package-file", "", "Path to location of package file (config: update.package.packageFile)") _ = packageCmd.MarkFlagRequired("package-file") return packageCmd @@ -38,7 +44,8 @@ SAP Integration Suite tenant.`, func runUpdatePackage(cmd *cobra.Command) error { log.Info().Msg("Executing update package command") - packageFile := config.GetString(cmd, "package-file") + // Support reading from config file under 'update.package' key + packageFile := config.GetStringWithFallback(cmd, "package-file", "update.package.packageFile") // Initialise HTTP executer serviceDetails := api.GetServiceDetails(cmd) diff --git a/internal/cmd/pd_common.go b/internal/cmd/pd_common.go new file mode 100644 index 0000000..ecc5f03 --- /dev/null +++ b/internal/cmd/pd_common.go @@ -0,0 +1,38 @@ +package cmd + +import ( + "github.com/engswee/flashpipe/internal/config" + "github.com/spf13/cobra" +) + +// Helper functions for Partner Directory commands to support reading +// configuration from both command-line flags and nested config file keys +// These are thin wrappers around the config package functions for backward compatibility + +// getConfigStringWithFallback reads a string value from command flag, +// falling back to a nested config key if the flag wasn't explicitly set +func getConfigStringWithFallback(cmd *cobra.Command, flagName, configKey string) string { + return config.GetStringWithFallback(cmd, flagName, configKey) +} + +// getConfigBoolWithFallback reads a bool value from command flag, +// falling back to a nested config key if the flag wasn't explicitly set +func getConfigBoolWithFallback(cmd *cobra.Command, flagName, configKey string) bool { + return config.GetBoolWithFallback(cmd, flagName, configKey) +} + +// getConfigStringSliceWithFallback reads a string slice value from command flag, +// falling back to a nested config key if the flag wasn't explicitly set +func getConfigStringSliceWithFallback(cmd *cobra.Command, flagName, configKey string) []string { + return config.GetStringSliceWithFallback(cmd, flagName, configKey) +} + +// contains checks if a string slice contains a specific string +func contains(slice []string, str string) bool { + for _, s := range slice { + if s == str { + return true + } + } + return false +} diff --git a/internal/cmd/pd_deploy.go b/internal/cmd/pd_deploy.go new file mode 100644 index 0000000..c29d1e2 --- /dev/null +++ b/internal/cmd/pd_deploy.go @@ -0,0 +1,499 @@ +package cmd + +import ( + "fmt" + "strings" + "time" + + "github.com/engswee/flashpipe/internal/analytics" + "github.com/engswee/flashpipe/internal/api" + "github.com/engswee/flashpipe/internal/repo" + "github.com/engswee/flashpipe/internal/str" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +func NewPDDeployCommand() *cobra.Command { + + pdDeployCmd := &cobra.Command{ + Use: "pd-deploy", + Short: "Deploy partner directory parameters to SAP CPI", + Long: `Upload all partner directory parameters from local files to SAP CPI. + +This command reads partner directory parameters from a local directory structure +and uploads them to the SAP CPI Partner Directory: + + {PID}/ + String.properties - String parameters as key=value pairs + Binary/ - Binary parameters as individual files + {ParamId}.{ext} - Binary parameter files + _metadata.json - Content type metadata + +The deploy operation supports several modes: + - Replace mode (default): Updates existing parameters with local values + - Add-only mode: Only creates new parameters, skips existing ones + - Full sync mode: Deletes remote parameters not present locally (local is source of truth) + +Authentication is performed using OAuth 2.0 client credentials flow or Basic Auth.`, + Example: ` # Deploy with OAuth (environment variables) + export FLASHPIPE_TMN_HOST="your-tenant.hana.ondemand.com" + export FLASHPIPE_OAUTH_HOST="your-tenant.authentication.eu10.hana.ondemand.com" + export FLASHPIPE_OAUTH_CLIENTID="your-client-id" + export FLASHPIPE_OAUTH_CLIENTSECRET="your-client-secret" + flashpipe pd-deploy + + # Deploy with explicit credentials and custom path + flashpipe pd-deploy \ + --tmn-host "your-tenant.hana.ondemand.com" \ + --oauth-host "your-tenant.authentication.eu10.hana.ondemand.com" \ + --oauth-clientid "your-client-id" \ + --oauth-clientsecret "your-client-secret" \ + --resources-path "./partner-directory" + + # Deploy in add-only mode (don't update existing parameters) + flashpipe pd-deploy --replace=false + + # Deploy with full sync (delete remote parameters not in local) + flashpipe pd-deploy --full-sync + + # Deploy only specific PIDs + flashpipe pd-deploy --pids "SAP_SYSTEM_001,CUSTOMER_API" + + # Dry run to see what would be changed + flashpipe pd-deploy --dry-run`, + RunE: func(cmd *cobra.Command, args []string) (err error) { + startTime := time.Now() + if err = runPDDeploy(cmd); err != nil { + cmd.SilenceUsage = true + } + analytics.Log(cmd, err, startTime) + return + }, + } + + // Define flags + // Note: These can be set in config file under 'pd-deploy' key + pdDeployCmd.Flags().String("resources-path", "./partner-directory", + "Path to partner directory parameters") + pdDeployCmd.Flags().Bool("replace", true, + "Replace existing values (false = add only missing values)") + pdDeployCmd.Flags().Bool("full-sync", false, + "Delete remote parameters not present locally (local is source of truth)") + pdDeployCmd.Flags().Bool("dry-run", false, + "Show what would be changed without making changes") + pdDeployCmd.Flags().StringSlice("pids", nil, + "Comma separated list of Partner IDs to deploy (e.g., 'PID1,PID2')") + + return pdDeployCmd +} + +func runPDDeploy(cmd *cobra.Command) error { + serviceDetails := api.GetServiceDetails(cmd) + + log.Info().Msg("Executing Partner Directory Deploy command") + + // Support reading from config file under 'pd-deploy' key + resourcesPath := getConfigStringWithFallback(cmd, "resources-path", "pd-deploy.resources-path") + replace := getConfigBoolWithFallback(cmd, "replace", "pd-deploy.replace") + fullSync := getConfigBoolWithFallback(cmd, "full-sync", "pd-deploy.full-sync") + dryRun := getConfigBoolWithFallback(cmd, "dry-run", "pd-deploy.dry-run") + pids := getConfigStringSliceWithFallback(cmd, "pids", "pd-deploy.pids") + + log.Info().Msgf("Resources Path: %s", resourcesPath) + log.Info().Msgf("Replace Mode: %v", replace) + log.Info().Msgf("Full Sync Mode: %v", fullSync) + log.Info().Msgf("Dry Run: %v", dryRun) + if len(pids) > 0 { + log.Info().Msgf("Filter PIDs: %v", pids) + } + + // Initialise HTTP executer + exe := api.InitHTTPExecuter(serviceDetails) + + // Initialise Partner Directory API + pdAPI := api.NewPartnerDirectory(exe) + + // Initialise Partner Directory Repository + pdRepo := repo.NewPartnerDirectory(resourcesPath) + + // Trim PIDs + pids = str.TrimSlice(pids) + + // Execute deploy + if err := deployPartnerDirectory(pdAPI, pdRepo, replace, fullSync, dryRun, pids); err != nil { + return err + } + + log.Info().Msg("🏆 Partner Directory Deploy completed successfully") + return nil +} + +func deployPartnerDirectory(pdAPI *api.PartnerDirectory, pdRepo *repo.PartnerDirectory, replace bool, fullSync bool, dryRun bool, pidsFilter []string) error { + log.Info().Msg("Starting Partner Directory Deploy...") + + // Get locally managed PIDs + managedPIDs, err := pdRepo.GetLocalPIDs() + if err != nil { + return fmt.Errorf("failed to get local PIDs: %w", err) + } + + // Filter managed PIDs if filter is specified + if len(pidsFilter) > 0 { + filteredPIDs := filterPIDs(managedPIDs, pidsFilter) + if len(filteredPIDs) == 0 { + return fmt.Errorf("no PIDs match the filter: %v", pidsFilter) + } + managedPIDs = filteredPIDs + log.Info().Msgf("Filtered to %d PIDs: %v", len(managedPIDs), managedPIDs) + } + + if fullSync && len(managedPIDs) > 0 { + log.Warn().Msg("Full sync will delete remote parameters not in local files!") + log.Warn().Msgf("Managed PIDs (only these will be affected):\n - %s", + strings.Join(managedPIDs, "\n - ")) + log.Warn().Msg("Parameters in other PIDs will NOT be touched.") + + if dryRun { + log.Info().Msg("DRY RUN MODE: No deletions will be performed") + } + } + + // Push string parameters + stringResults, err := deployStringParameters(pdAPI, pdRepo, replace, dryRun, pidsFilter) + if err != nil { + return fmt.Errorf("failed to deploy string parameters: %w", err) + } + + // Push binary parameters + binaryResults, err := deployBinaryParameters(pdAPI, pdRepo, replace, dryRun, pidsFilter) + if err != nil { + return fmt.Errorf("failed to deploy binary parameters: %w", err) + } + + // Full sync - delete remote entries not in local (only for managed PIDs) + var deletionResults *api.BatchResult + if fullSync && !dryRun { + log.Info().Msg("Executing full sync - deleting remote entries not present locally...") + deletionResults, err = deleteRemoteEntriesNotInLocal(pdAPI, pdRepo, managedPIDs) + if err != nil { + log.Warn().Msgf("Error during full sync deletion: %v", err) + } else { + log.Info().Msgf("Parameters Deleted: %d", len(deletionResults.Deleted)) + if len(deletionResults.Deleted) > 0 { + log.Info().Msg("Deleted parameters:") + for _, deleted := range deletionResults.Deleted { + log.Info().Msgf(" - %s", deleted) + } + } + if len(deletionResults.Errors) > 0 { + log.Info().Msgf("Deletion Errors: %d", len(deletionResults.Errors)) + for _, err := range deletionResults.Errors { + log.Warn().Msg(err) + } + } + } + } else if fullSync && dryRun { + log.Info().Msg("DRY RUN: Would execute full sync deletion") + log.Warn().Msgf("Would delete remote parameters not in local for PIDs:):\n - %s", + strings.Join(managedPIDs, "\n - ")) + } + + // Log summary + log.Info().Msgf("String Parameters - Created: %d, Updated: %d, Unchanged: %d, Errors: %d", + len(stringResults.Created), len(stringResults.Updated), len(stringResults.Unchanged), len(stringResults.Errors)) + log.Info().Msgf("Binary Parameters - Created: %d, Updated: %d, Unchanged: %d, Errors: %d", + len(binaryResults.Created), len(binaryResults.Updated), len(binaryResults.Unchanged), len(binaryResults.Errors)) + + if fullSync && deletionResults != nil { + log.Info().Msgf("Full Sync - Deleted: %d, Errors: %d", + len(deletionResults.Deleted), len(deletionResults.Errors)) + if len(deletionResults.Deleted) > 0 { + log.Info().Msgf("Deleted: %s", strings.Join(deletionResults.Deleted, ", ")) + } + } + + if len(stringResults.Errors) > 0 || len(binaryResults.Errors) > 0 { + log.Warn().Msg("Errors encountered during deploy:") + for _, err := range stringResults.Errors { + log.Warn().Msgf("String: %s", err) + } + for _, err := range binaryResults.Errors { + log.Warn().Msgf("Binary: %s", err) + } + } + + if dryRun { + log.Info().Msg("DRY RUN completed - no changes were made!") + } + + return nil +} + +func deployStringParameters(pdAPI *api.PartnerDirectory, pdRepo *repo.PartnerDirectory, replace bool, dryRun bool, pidsFilter []string) (*api.BatchResult, error) { + log.Debug().Msg("Loading string parameters from local files") + + // Get local PIDs + localPIDs, err := pdRepo.GetLocalPIDs() + if err != nil { + return nil, err + } + + // Filter if needed + if len(pidsFilter) > 0 { + localPIDs = filterPIDs(localPIDs, pidsFilter) + } + + results := &api.BatchResult{ + Created: []string{}, + Updated: []string{}, + Unchanged: []string{}, + Errors: []string{}, + } + + // Load and deploy parameters for each PID + for _, pid := range localPIDs { + parameters, err := pdRepo.ReadStringParameters(pid) + if err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("Failed to read %s: %v", pid, err)) + continue + } + + for _, param := range parameters { + key := fmt.Sprintf("%s/%s", param.Pid, param.ID) + + if dryRun { + // Just check if it exists and report what would happen + existing, err := pdAPI.GetStringParameter(param.Pid, param.ID) + if err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + continue + } + + if existing == nil { + results.Created = append(results.Created, key) + log.Info().Msgf("[DRY RUN] Would create: %s", key) + } else if replace && existing.Value != param.Value { + results.Updated = append(results.Updated, key) + log.Info().Msgf("[DRY RUN] Would update: %s", key) + } else { + results.Unchanged = append(results.Unchanged, key) + } + continue + } + + // Check if parameter exists + existing, err := pdAPI.GetStringParameter(param.Pid, param.ID) + if err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + continue + } + + if existing == nil { + // Create new parameter + if err := pdAPI.CreateStringParameter(param); err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + } else { + results.Created = append(results.Created, key) + log.Debug().Msgf("Created: %s", key) + } + } else if replace && existing.Value != param.Value { + // Update existing parameter + if err := pdAPI.UpdateStringParameter(param); err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + } else { + results.Updated = append(results.Updated, key) + log.Debug().Msgf("Updated: %s", key) + } + } else { + results.Unchanged = append(results.Unchanged, key) + } + } + } + + return results, nil +} + +func deployBinaryParameters(pdAPI *api.PartnerDirectory, pdRepo *repo.PartnerDirectory, replace bool, dryRun bool, pidsFilter []string) (*api.BatchResult, error) { + log.Debug().Msg("Loading binary parameters from local files") + + // Get local PIDs + localPIDs, err := pdRepo.GetLocalPIDs() + if err != nil { + return nil, err + } + + // Filter if needed + if len(pidsFilter) > 0 { + localPIDs = filterPIDs(localPIDs, pidsFilter) + } + + results := &api.BatchResult{ + Created: []string{}, + Updated: []string{}, + Unchanged: []string{}, + Errors: []string{}, + } + + // Load and deploy parameters for each PID + for _, pid := range localPIDs { + parameters, err := pdRepo.ReadBinaryParameters(pid) + if err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("Failed to read %s: %v", pid, err)) + continue + } + + for _, param := range parameters { + key := fmt.Sprintf("%s/%s", param.Pid, param.ID) + + if dryRun { + // Just check if it exists and report what would happen + existing, err := pdAPI.GetBinaryParameter(param.Pid, param.ID) + if err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + continue + } + + if existing == nil { + results.Created = append(results.Created, key) + log.Info().Msgf("[DRY RUN] Would create: %s", key) + } else if replace && existing.Value != param.Value { + results.Updated = append(results.Updated, key) + log.Info().Msgf("[DRY RUN] Would update: %s", key) + } else { + results.Unchanged = append(results.Unchanged, key) + } + continue + } + + // Check if parameter exists + existing, err := pdAPI.GetBinaryParameter(param.Pid, param.ID) + if err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + continue + } + + if existing == nil { + // Create new parameter + if err := pdAPI.CreateBinaryParameter(param); err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + } else { + results.Created = append(results.Created, key) + log.Debug().Msgf("Created: %s", key) + } + } else if replace && existing.Value != param.Value { + // Update existing parameter + if err := pdAPI.UpdateBinaryParameter(param); err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("%s: %v", key, err)) + } else { + results.Updated = append(results.Updated, key) + log.Debug().Msgf("Updated: %s", key) + } + } else { + results.Unchanged = append(results.Unchanged, key) + } + } + } + + return results, nil +} + +func deleteRemoteEntriesNotInLocal(pdAPI *api.PartnerDirectory, pdRepo *repo.PartnerDirectory, managedPIDs []string) (*api.BatchResult, error) { + results := &api.BatchResult{ + Deleted: []string{}, + Errors: []string{}, + } + + // Load local parameters for managed PIDs + localStringParams := make(map[string]map[string]bool) // PID -> ID -> exists + localBinaryParams := make(map[string]map[string]bool) + + for _, pid := range managedPIDs { + // Load string parameters + stringParams, err := pdRepo.ReadStringParameters(pid) + if err != nil { + log.Warn().Msgf("Failed to read string parameters for PID %s: %v", pid, err) + } else { + if localStringParams[pid] == nil { + localStringParams[pid] = make(map[string]bool) + } + for _, param := range stringParams { + localStringParams[pid][param.ID] = true + } + } + + // Load binary parameters + binaryParams, err := pdRepo.ReadBinaryParameters(pid) + if err != nil { + log.Warn().Msgf("Failed to read binary parameters for PID %s: %v", pid, err) + } else { + if localBinaryParams[pid] == nil { + localBinaryParams[pid] = make(map[string]bool) + } + for _, param := range binaryParams { + localBinaryParams[pid][param.ID] = true + } + } + } + + // Get all remote string parameters + remoteStringParams, err := pdAPI.GetStringParameters("Pid,Id") + if err != nil { + return nil, fmt.Errorf("failed to get remote string parameters: %w", err) + } + + // Delete string parameters not in local for managed PIDs + for _, param := range remoteStringParams { + if !contains(managedPIDs, param.Pid) { + continue // Skip PIDs we don't manage + } + + if localStringParams[param.Pid] == nil || !localStringParams[param.Pid][param.ID] { + key := fmt.Sprintf("%s/%s", param.Pid, param.ID) + if err := pdAPI.DeleteStringParameter(param.Pid, param.ID); err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("Failed to delete string %s: %v", key, err)) + } else { + results.Deleted = append(results.Deleted, key) + log.Debug().Msgf("Deleted string parameter: %s", key) + } + } + } + + // Get all remote binary parameters + remoteBinaryParams, err := pdAPI.GetBinaryParameters("Pid,Id") + if err != nil { + return nil, fmt.Errorf("failed to get remote binary parameters: %w", err) + } + + // Delete binary parameters not in local for managed PIDs + for _, param := range remoteBinaryParams { + if !contains(managedPIDs, param.Pid) { + continue // Skip PIDs we don't manage + } + + if localBinaryParams[param.Pid] == nil || !localBinaryParams[param.Pid][param.ID] { + key := fmt.Sprintf("%s/%s", param.Pid, param.ID) + if err := pdAPI.DeleteBinaryParameter(param.Pid, param.ID); err != nil { + results.Errors = append(results.Errors, fmt.Sprintf("Failed to delete binary %s: %v", key, err)) + } else { + results.Deleted = append(results.Deleted, key) + log.Debug().Msgf("Deleted binary parameter: %s", key) + } + } + } + + return results, nil +} + +func filterPIDs(pids []string, filter []string) []string { + if len(filter) == 0 { + return pids + } + + result := make([]string, 0) + for _, pid := range pids { + if contains(filter, pid) { + result = append(result, pid) + } + } + return result +} diff --git a/internal/cmd/pd_snapshot.go b/internal/cmd/pd_snapshot.go new file mode 100644 index 0000000..9f9eb93 --- /dev/null +++ b/internal/cmd/pd_snapshot.go @@ -0,0 +1,225 @@ +package cmd + +import ( + "fmt" + "time" + + "github.com/engswee/flashpipe/internal/analytics" + "github.com/engswee/flashpipe/internal/api" + "github.com/engswee/flashpipe/internal/repo" + "github.com/engswee/flashpipe/internal/str" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +func NewPDSnapshotCommand() *cobra.Command { + + pdSnapshotCmd := &cobra.Command{ + Use: "pd-snapshot", + Short: "Download partner directory parameters from SAP CPI", + Long: `Download all partner directory parameters from SAP CPI and save them locally. + +This command retrieves both string and binary parameters from the SAP CPI Partner Directory +and organizes them in a local directory structure: + + {PID}/ + String.properties - String parameters as key=value pairs + Binary/ - Binary parameters as individual files + {ParamId}.{ext} - Binary parameter files + _metadata.json - Content type metadata + +The snapshot operation supports two modes: + - Replace mode (default): Overwrites existing local files + - Add-only mode: Only adds new parameters, preserves existing values + +Authentication is performed using OAuth 2.0 client credentials flow or Basic Auth.`, + Example: ` # Snapshot with OAuth (environment variables) + export FLASHPIPE_TMN_HOST="your-tenant.hana.ondemand.com" + export FLASHPIPE_OAUTH_HOST="your-tenant.authentication.eu10.hana.ondemand.com" + export FLASHPIPE_OAUTH_CLIENTID="your-client-id" + export FLASHPIPE_OAUTH_CLIENTSECRET="your-client-secret" + flashpipe pd-snapshot + + # Snapshot with explicit credentials and custom path + flashpipe pd-snapshot \ + --tmn-host "your-tenant.hana.ondemand.com" \ + --oauth-host "your-tenant.authentication.eu10.hana.ondemand.com" \ + --oauth-clientid "your-client-id" \ + --oauth-clientsecret "your-client-secret" \ + --resources-path "./partner-directory" + + # Snapshot in add-only mode (don't overwrite existing values) + flashpipe pd-snapshot --replace=false + + # Snapshot only specific PIDs + flashpipe pd-snapshot --pids "SAP_SYSTEM_001,CUSTOMER_API"`, + RunE: func(cmd *cobra.Command, args []string) (err error) { + startTime := time.Now() + if err = runPDSnapshot(cmd); err != nil { + cmd.SilenceUsage = true + } + analytics.Log(cmd, err, startTime) + return + }, + } + + // Define flags + // Note: These can be set in config file under 'pd-snapshot' key + pdSnapshotCmd.Flags().String("resources-path", "./partner-directory", + "Path to save partner directory parameters") + pdSnapshotCmd.Flags().Bool("replace", true, + "Replace existing values (false = add only missing values)") + pdSnapshotCmd.Flags().StringSlice("pids", nil, + "Comma separated list of Partner IDs to snapshot (e.g., 'PID1,PID2')") + + return pdSnapshotCmd +} + +func runPDSnapshot(cmd *cobra.Command) error { + serviceDetails := api.GetServiceDetails(cmd) + + log.Info().Msg("Executing Partner Directory Snapshot command") + + // Support reading from config file under 'pd-snapshot' key + resourcesPath := getConfigStringWithFallback(cmd, "resources-path", "pd-snapshot.resources-path") + replace := getConfigBoolWithFallback(cmd, "replace", "pd-snapshot.replace") + pids := getConfigStringSliceWithFallback(cmd, "pids", "pd-snapshot.pids") + + log.Info().Msgf("Resources Path: %s", resourcesPath) + log.Info().Msgf("Replace Mode: %v", replace) + if len(pids) > 0 { + log.Info().Msgf("Filter PIDs: %v", pids) + } + + // Trim PIDs + pids = str.TrimSlice(pids) + + // Initialise HTTP executer + exe := api.InitHTTPExecuter(serviceDetails) + + // Initialise Partner Directory API + pdAPI := api.NewPartnerDirectory(exe) + + // Initialise Partner Directory Repository + pdRepo := repo.NewPartnerDirectory(resourcesPath) + + // Execute snapshot + if err := snapshotPartnerDirectory(pdAPI, pdRepo, replace, pids); err != nil { + return err + } + + log.Info().Msg("🏆 Partner Directory Snapshot completed successfully") + return nil +} + +func snapshotPartnerDirectory(pdAPI *api.PartnerDirectory, pdRepo *repo.PartnerDirectory, replace bool, pidsFilter []string) error { + log.Info().Msg("Starting Partner Directory Snapshot...") + + // Download string parameters + stringCount, err := snapshotStringParameters(pdAPI, pdRepo, replace, pidsFilter) + if err != nil { + return fmt.Errorf("failed to download string parameters: %w", err) + } + log.Info().Msgf("Downloaded %d string parameters", stringCount) + + // Download binary parameters + binaryCount, err := snapshotBinaryParameters(pdAPI, pdRepo, replace, pidsFilter) + if err != nil { + return fmt.Errorf("failed to download binary parameters: %w", err) + } + log.Info().Msgf("Downloaded %d binary parameters", binaryCount) + + return nil +} + +func snapshotStringParameters(pdAPI *api.PartnerDirectory, pdRepo *repo.PartnerDirectory, replace bool, pidsFilter []string) (int, error) { + log.Debug().Msg("Fetching string parameters from Partner Directory") + + parameters, err := pdAPI.GetStringParameters("Pid,Id,Value") + if err != nil { + return 0, err + } + + // Filter by PIDs if specified + if len(pidsFilter) > 0 { + filtered := make([]api.StringParameter, 0) + for _, param := range parameters { + if contains(pidsFilter, param.Pid) { + filtered = append(filtered, param) + } + } + parameters = filtered + } + + log.Debug().Msgf("Fetched %d string parameters from Partner Directory", len(parameters)) + + // Group by PID + paramsByPid := make(map[string][]api.StringParameter) + for _, param := range parameters { + paramsByPid[param.Pid] = append(paramsByPid[param.Pid], param) + } + + // Log all PIDs found + pids := make([]string, 0, len(paramsByPid)) + for pid := range paramsByPid { + pids = append(pids, pid) + } + log.Info().Msgf("Found %d Partner IDs with string parameters: %v", len(pids), pids) + + // Process each PID + for pid, pidParams := range paramsByPid { + log.Info().Msgf("Processing string parameters for PID: %s (%d parameters)", pid, len(pidParams)) + + if err := pdRepo.WriteStringParameters(pid, pidParams, replace); err != nil { + return 0, fmt.Errorf("failed to write string parameters for PID %s: %w", pid, err) + } + } + + return len(parameters), nil +} + +func snapshotBinaryParameters(pdAPI *api.PartnerDirectory, pdRepo *repo.PartnerDirectory, replace bool, pidsFilter []string) (int, error) { + log.Debug().Msg("Fetching binary parameters from Partner Directory") + + parameters, err := pdAPI.GetBinaryParameters("") + if err != nil { + return 0, err + } + + // Filter by PIDs if specified + if len(pidsFilter) > 0 { + filtered := make([]api.BinaryParameter, 0) + for _, param := range parameters { + if contains(pidsFilter, param.Pid) { + filtered = append(filtered, param) + } + } + parameters = filtered + } + + log.Debug().Msgf("Fetched %d binary parameters from Partner Directory", len(parameters)) + + // Group by PID + paramsByPid := make(map[string][]api.BinaryParameter) + for _, param := range parameters { + paramsByPid[param.Pid] = append(paramsByPid[param.Pid], param) + } + + // Log all PIDs found + pids := make([]string, 0, len(paramsByPid)) + for pid := range paramsByPid { + pids = append(pids, pid) + } + log.Info().Msgf("Found %d Partner IDs with binary parameters: %v", len(pids), pids) + + // Process each PID + for pid, pidParams := range paramsByPid { + log.Info().Msgf("Processing binary parameters for PID: %s (%d parameters)", pid, len(pidParams)) + + if err := pdRepo.WriteBinaryParameters(pid, pidParams, replace); err != nil { + return 0, fmt.Errorf("failed to write binary parameters for PID %s: %w", pid, err) + } + } + + return len(parameters), nil +} diff --git a/internal/cmd/restore.go b/internal/cmd/restore.go index 258ee90..baae61b 100644 --- a/internal/cmd/restore.go +++ b/internal/cmd/restore.go @@ -2,6 +2,11 @@ package cmd import ( "fmt" + "os" + "path/filepath" + "strings" + "time" + "github.com/engswee/flashpipe/internal/analytics" "github.com/engswee/flashpipe/internal/api" "github.com/engswee/flashpipe/internal/config" @@ -11,10 +16,6 @@ import ( "github.com/go-errors/errors" "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "os" - "path/filepath" - "strings" - "time" ) func NewRestoreCommand() *cobra.Command { @@ -22,17 +23,21 @@ func NewRestoreCommand() *cobra.Command { restoreCmd := &cobra.Command{ Use: "restore", Short: "Restore integration packages from Git to tenant", - Long: `Restore all editable integration packages from a Git repository to SAP Integration Suite tenant.`, + Long: `Restore all editable integration packages from a Git repository to SAP Integration Suite tenant. + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'restore' section. CLI flags override config file settings.`, PreRunE: func(cmd *cobra.Command, args []string) error { // If artifacts directory is provided, validate that is it a subdirectory of Git repo - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "restore.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } if gitRepoDir != "" { - artifactsDir, err := config.GetStringWithEnvExpand(cmd, "dir-artifacts") + artifactsDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-artifacts", "restore.dirArtifacts") if err != nil { return fmt.Errorf("security alert for --dir-artifacts: %w", err) } @@ -59,20 +64,21 @@ func NewRestoreCommand() *cobra.Command { func runRestore(cmd *cobra.Command) error { log.Info().Msg("Executing snapshot restore command") - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + // Support reading from config file under 'restore' key + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "restore.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } - artifactsBaseDir, err := config.GetStringWithEnvExpandWithDefault(cmd, "dir-artifacts", gitRepoDir) - if err != nil { - return fmt.Errorf("security alert for --dir-artifacts: %w", err) + artifactsBaseDir := config.GetStringWithFallback(cmd, "dir-artifacts", "restore.dirArtifacts") + if artifactsBaseDir == "" { + artifactsBaseDir = gitRepoDir } - workDir, err := config.GetStringWithEnvExpand(cmd, "dir-work") + workDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-work", "restore.dirWork") if err != nil { return fmt.Errorf("security alert for --dir-work: %w", err) } - includedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-include")) - excludedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-exclude")) + includedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-include", "restore.idsInclude")) + excludedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-exclude", "restore.idsExclude")) serviceDetails := api.GetServiceDetails(cmd) err = restoreSnapshot(serviceDetails, artifactsBaseDir, workDir, includedIds, excludedIds) diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 246c5b3..59e6c08 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -24,7 +24,7 @@ func NewCmdRoot() *cobra.Command { Long: `FlashPipe - The CI/CD Companion for SAP Integration Suite FlashPipe is a CLI that is used to simplify the Build-To-Deploy cycle -for SAP Integration Suite by providing CI/CD capabilities for +for SAP Integration Suite by providing CI/CD capabilities for automating time-consuming manual tasks like: - synchronising integration artifacts to Git - creating/updating integration artifacts to SAP Integration Suite @@ -73,6 +73,11 @@ func Execute() { snapshotCmd := NewSnapshotCommand() snapshotCmd.AddCommand(NewRestoreCommand()) rootCmd.AddCommand(snapshotCmd) + rootCmd.AddCommand(NewPDSnapshotCommand()) + rootCmd.AddCommand(NewPDDeployCommand()) + rootCmd.AddCommand(NewConfigGenerateCommand()) + rootCmd.AddCommand(NewFlashpipeOrchestratorCommand()) + rootCmd.AddCommand(NewConfigureCommand()) err := rootCmd.Execute() diff --git a/internal/cmd/snapshot.go b/internal/cmd/snapshot.go index cd57065..ffb124a 100644 --- a/internal/cmd/snapshot.go +++ b/internal/cmd/snapshot.go @@ -23,23 +23,27 @@ func NewSnapshotCommand() *cobra.Command { Use: "snapshot", Short: "Snapshot integration packages from tenant to Git", Long: `Snapshot all editable integration packages from SAP Integration Suite -tenant to a Git repository.`, +tenant to a Git repository. + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'snapshot' section. CLI flags override config file settings.`, PreRunE: func(cmd *cobra.Command, args []string) error { // Validate Draft Handling - draftHandling := config.GetString(cmd, "draft-handling") + draftHandling := config.GetStringWithFallback(cmd, "draft-handling", "snapshot.draftHandling") switch draftHandling { case "SKIP", "ADD", "ERROR": default: return fmt.Errorf("invalid value for --draft-handling = %v", draftHandling) } // If artifacts directory is provided, validate that is it a subdirectory of Git repo - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "snapshot.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } if gitRepoDir != "" { - artifactsDir, err := config.GetStringWithEnvExpand(cmd, "dir-artifacts") + artifactsDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-artifacts", "snapshot.dirArtifacts") if err != nil { return fmt.Errorf("security alert for --dir-artifacts: %w", err) } @@ -61,18 +65,19 @@ tenant to a Git repository.`, } // Define cobra flags, the default value has the lowest (least significant) precedence - snapshotCmd.PersistentFlags().String("dir-git-repo", "", "Directory of Git repository") - snapshotCmd.PersistentFlags().String("dir-artifacts", "", "Directory containing contents of artifacts (grouped into packages)") - snapshotCmd.PersistentFlags().String("dir-work", "/tmp", "Working directory for in-transit files") - snapshotCmd.Flags().String("draft-handling", "SKIP", "Handling when artifact is in draft version. Allowed values: SKIP, ADD, ERROR") - snapshotCmd.PersistentFlags().StringSlice("ids-include", nil, "List of included package IDs") - snapshotCmd.PersistentFlags().StringSlice("ids-exclude", nil, "List of excluded package IDs") - - snapshotCmd.Flags().String("git-commit-msg", "Tenant snapshot of "+time.Now().Format(time.UnixDate), "Message used in commit") - snapshotCmd.Flags().String("git-commit-user", "github-actions[bot]", "User used in commit") - snapshotCmd.Flags().String("git-commit-email", "41898282+github-actions[bot]@users.noreply.github.com", "Email used in commit") - snapshotCmd.Flags().Bool("git-skip-commit", false, "Skip committing changes to Git repository") - snapshotCmd.Flags().Bool("sync-package-details", true, "Sync details of Integration Packages") + // Note: These can be set in config file under 'snapshot' key + snapshotCmd.PersistentFlags().String("dir-git-repo", "", "Directory of Git repository (config: snapshot.dirGitRepo)") + snapshotCmd.PersistentFlags().String("dir-artifacts", "", "Directory containing contents of artifacts (grouped into packages) (config: snapshot.dirArtifacts)") + snapshotCmd.PersistentFlags().String("dir-work", "/tmp", "Working directory for in-transit files (config: snapshot.dirWork)") + snapshotCmd.Flags().String("draft-handling", "SKIP", "Handling when artifact is in draft version. Allowed values: SKIP, ADD, ERROR (config: snapshot.draftHandling)") + snapshotCmd.PersistentFlags().StringSlice("ids-include", nil, "List of included package IDs (config: snapshot.idsInclude)") + snapshotCmd.PersistentFlags().StringSlice("ids-exclude", nil, "List of excluded package IDs (config: snapshot.idsExclude)") + + snapshotCmd.Flags().String("git-commit-msg", "Tenant snapshot of "+time.Now().Format(time.UnixDate), "Message used in commit (config: snapshot.gitCommitMsg)") + snapshotCmd.Flags().String("git-commit-user", "github-actions[bot]", "User used in commit (config: snapshot.gitCommitUser)") + snapshotCmd.Flags().String("git-commit-email", "41898282+github-actions[bot]@users.noreply.github.com", "Email used in commit (config: snapshot.gitCommitEmail)") + snapshotCmd.Flags().Bool("git-skip-commit", false, "Skip committing changes to Git repository (config: snapshot.gitSkipCommit)") + snapshotCmd.Flags().Bool("sync-package-details", true, "Sync details of Integration Packages (config: snapshot.syncPackageDetails)") _ = snapshotCmd.MarkFlagRequired("dir-git-repo") snapshotCmd.MarkFlagsMutuallyExclusive("ids-include", "ids-exclude") @@ -83,26 +88,27 @@ tenant to a Git repository.`, func runSnapshot(cmd *cobra.Command) error { log.Info().Msg("Executing snapshot command") - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + // Support reading from config file under 'snapshot' key + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "snapshot.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } - artifactsBaseDir, err := config.GetStringWithEnvExpandWithDefault(cmd, "dir-artifacts", gitRepoDir) - if err != nil { - return fmt.Errorf("security alert for --dir-artifacts: %w", err) + artifactsBaseDir := config.GetStringWithFallback(cmd, "dir-artifacts", "snapshot.dirArtifacts") + if artifactsBaseDir == "" { + artifactsBaseDir = gitRepoDir } - workDir, err := config.GetStringWithEnvExpand(cmd, "dir-work") + workDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-work", "snapshot.dirWork") if err != nil { return fmt.Errorf("security alert for --dir-work: %w", err) } - draftHandling := config.GetString(cmd, "draft-handling") - includedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-include")) - excludedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-exclude")) - commitMsg := config.GetString(cmd, "git-commit-msg") - commitUser := config.GetString(cmd, "git-commit-user") - commitEmail := config.GetString(cmd, "git-commit-email") - skipCommit := config.GetBool(cmd, "git-skip-commit") - syncPackageLevelDetails := config.GetBool(cmd, "sync-package-details") + draftHandling := config.GetStringWithFallback(cmd, "draft-handling", "snapshot.draftHandling") + includedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-include", "snapshot.idsInclude")) + excludedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-exclude", "snapshot.idsExclude")) + commitMsg := config.GetStringWithFallback(cmd, "git-commit-msg", "snapshot.gitCommitMsg") + commitUser := config.GetStringWithFallback(cmd, "git-commit-user", "snapshot.gitCommitUser") + commitEmail := config.GetStringWithFallback(cmd, "git-commit-email", "snapshot.gitCommitEmail") + skipCommit := config.GetBoolWithFallback(cmd, "git-skip-commit", "snapshot.gitSkipCommit") + syncPackageLevelDetails := config.GetBoolWithFallback(cmd, "sync-package-details", "snapshot.syncPackageDetails") serviceDetails := api.GetServiceDetails(cmd) err = getTenantSnapshot(serviceDetails, artifactsBaseDir, workDir, draftHandling, syncPackageLevelDetails, includedIds, excludedIds) diff --git a/internal/cmd/sync.go b/internal/cmd/sync.go index 414f5cb..204ecd2 100644 --- a/internal/cmd/sync.go +++ b/internal/cmd/sync.go @@ -24,29 +24,33 @@ func NewSyncCommand() *cobra.Command { Use: "sync", Short: "Sync designtime artifacts between tenant and Git", Long: `Synchronise designtime artifacts between SAP Integration Suite -tenant and a Git repository.`, +tenant and a Git repository. + +Configuration: + Settings can be loaded from the global config file (--config) under the + 'sync' section. CLI flags override config file settings.`, PreRunE: func(cmd *cobra.Command, args []string) error { // Validate Directory Naming Type - dirNamingType := config.GetString(cmd, "dir-naming-type") + dirNamingType := config.GetStringWithFallback(cmd, "dir-naming-type", "sync.dirNamingType") switch dirNamingType { case "ID", "NAME": default: return fmt.Errorf("invalid value for --dir-naming-type = %v", dirNamingType) } // Validate Draft Handling - draftHandling := config.GetString(cmd, "draft-handling") + draftHandling := config.GetStringWithFallback(cmd, "draft-handling", "sync.draftHandling") switch draftHandling { case "SKIP", "ADD", "ERROR": default: return fmt.Errorf("invalid value for --draft-handling = %v", draftHandling) } // If artifacts directory is provided, validate that is it a subdirectory of Git repo - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "sync.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } if gitRepoDir != "" { - artifactsDir, err := config.GetStringWithEnvExpand(cmd, "dir-artifacts") + artifactsDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-artifacts", "sync.dirArtifacts") if err != nil { return fmt.Errorf("security alert for --dir-artifacts: %w", err) } @@ -56,7 +60,7 @@ tenant and a Git repository.`, } } // Validate target - target := config.GetString(cmd, "target") + target := config.GetStringWithFallback(cmd, "target", "sync.target") switch target { case "git", "tenant": default: @@ -75,21 +79,22 @@ tenant and a Git repository.`, } // Define cobra flags, the default value has the lowest (least significant) precedence - syncCmd.Flags().String("package-id", "", "ID of Integration Package") - syncCmd.PersistentFlags().String("dir-git-repo", "", "Directory of Git repository") - syncCmd.PersistentFlags().String("dir-artifacts", "", "Directory containing contents of artifacts") - syncCmd.PersistentFlags().String("dir-work", "/tmp", "Working directory for in-transit files") - syncCmd.Flags().String("dir-naming-type", "ID", "Name artifact directory by ID or Name. Allowed values: ID, NAME") - syncCmd.Flags().String("draft-handling", "SKIP", "Handling when artifact is in draft version. Allowed values: SKIP, ADD, ERROR") - syncCmd.PersistentFlags().StringSlice("ids-include", nil, "List of included artifact IDs") - syncCmd.PersistentFlags().StringSlice("ids-exclude", nil, "List of excluded artifact IDs") - syncCmd.PersistentFlags().String("target", "git", "Target of sync. Allowed values: git, tenant") - syncCmd.PersistentFlags().String("git-commit-msg", "Sync repo from tenant", "Message used in commit") - syncCmd.PersistentFlags().String("git-commit-user", "github-actions[bot]", "User used in commit") - syncCmd.PersistentFlags().String("git-commit-email", "41898282+github-actions[bot]@users.noreply.github.com", "Email used in commit") - syncCmd.Flags().StringSlice("script-collection-map", nil, "Comma-separated source-target ID pairs for converting script collection references during sync ") - syncCmd.PersistentFlags().Bool("git-skip-commit", false, "Skip committing changes to Git repository") - syncCmd.Flags().Bool("sync-package-details", false, "Sync details of Integration Package") + // Note: These can be set in config file under 'sync' key + syncCmd.Flags().String("package-id", "", "ID of Integration Package (config: sync.packageId)") + syncCmd.PersistentFlags().String("dir-git-repo", "", "Directory of Git repository (config: sync.dirGitRepo)") + syncCmd.PersistentFlags().String("dir-artifacts", "", "Directory containing contents of artifacts (config: sync.dirArtifacts)") + syncCmd.PersistentFlags().String("dir-work", "/tmp", "Working directory for in-transit files (config: sync.dirWork)") + syncCmd.Flags().String("dir-naming-type", "ID", "Name artifact directory by ID or Name. Allowed values: ID, NAME (config: sync.dirNamingType)") + syncCmd.Flags().String("draft-handling", "SKIP", "Handling when artifact is in draft version. Allowed values: SKIP, ADD, ERROR (config: sync.draftHandling)") + syncCmd.PersistentFlags().StringSlice("ids-include", nil, "List of included artifact IDs (config: sync.idsInclude)") + syncCmd.PersistentFlags().StringSlice("ids-exclude", nil, "List of excluded artifact IDs (config: sync.idsExclude)") + syncCmd.PersistentFlags().String("target", "git", "Target of sync. Allowed values: git, tenant (config: sync.target)") + syncCmd.PersistentFlags().String("git-commit-msg", "Sync repo from tenant", "Message used in commit (config: sync.gitCommitMsg)") + syncCmd.PersistentFlags().String("git-commit-user", "github-actions[bot]", "User used in commit (config: sync.gitCommitUser)") + syncCmd.PersistentFlags().String("git-commit-email", "41898282+github-actions[bot]@users.noreply.github.com", "Email used in commit (config: sync.gitCommitEmail)") + syncCmd.Flags().StringSlice("script-collection-map", nil, "Comma-separated source-target ID pairs for converting script collection references during sync (config: sync.scriptCollectionMap)") + syncCmd.PersistentFlags().Bool("git-skip-commit", false, "Skip committing changes to Git repository (config: sync.gitSkipCommit)") + syncCmd.Flags().Bool("sync-package-details", false, "Sync details of Integration Package (config: sync.syncPackageDetails)") _ = syncCmd.MarkFlagRequired("package-id") _ = syncCmd.MarkFlagRequired("dir-git-repo") @@ -101,30 +106,31 @@ tenant and a Git repository.`, func runSync(cmd *cobra.Command) error { log.Info().Msg("Executing sync command") - packageId := config.GetString(cmd, "package-id") - gitRepoDir, err := config.GetStringWithEnvExpand(cmd, "dir-git-repo") + // Support reading from config file under 'sync' key + packageId := config.GetStringWithFallback(cmd, "package-id", "sync.packageId") + gitRepoDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-git-repo", "sync.dirGitRepo") if err != nil { return fmt.Errorf("security alert for --dir-git-repo: %w", err) } - artifactsDir, err := config.GetStringWithEnvExpandWithDefault(cmd, "dir-artifacts", gitRepoDir) - if err != nil { - return fmt.Errorf("security alert for --dir-artifacts: %w", err) + artifactsDir := config.GetStringWithFallback(cmd, "dir-artifacts", "sync.dirArtifacts") + if artifactsDir == "" { + artifactsDir = gitRepoDir } - workDir, err := config.GetStringWithEnvExpand(cmd, "dir-work") + workDir, err := config.GetStringWithEnvExpandAndFallback(cmd, "dir-work", "sync.dirWork") if err != nil { return fmt.Errorf("security alert for --dir-work: %w", err) } - dirNamingType := config.GetString(cmd, "dir-naming-type") - draftHandling := config.GetString(cmd, "draft-handling") - includedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-include")) - excludedIds := str.TrimSlice(config.GetStringSlice(cmd, "ids-exclude")) - commitMsg := config.GetString(cmd, "git-commit-msg") - commitUser := config.GetString(cmd, "git-commit-user") - commitEmail := config.GetString(cmd, "git-commit-email") - scriptCollectionMap := str.TrimSlice(config.GetStringSlice(cmd, "script-collection-map")) - skipCommit := config.GetBool(cmd, "git-skip-commit") - syncPackageLevelDetails := config.GetBool(cmd, "sync-package-details") - target := config.GetString(cmd, "target") + dirNamingType := config.GetStringWithFallback(cmd, "dir-naming-type", "sync.dirNamingType") + draftHandling := config.GetStringWithFallback(cmd, "draft-handling", "sync.draftHandling") + includedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-include", "sync.idsInclude")) + excludedIds := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "ids-exclude", "sync.idsExclude")) + commitMsg := config.GetStringWithFallback(cmd, "git-commit-msg", "sync.gitCommitMsg") + commitUser := config.GetStringWithFallback(cmd, "git-commit-user", "sync.gitCommitUser") + commitEmail := config.GetStringWithFallback(cmd, "git-commit-email", "sync.gitCommitEmail") + scriptCollectionMap := str.TrimSlice(config.GetStringSliceWithFallback(cmd, "script-collection-map", "sync.scriptCollectionMap")) + skipCommit := config.GetBoolWithFallback(cmd, "git-skip-commit", "sync.gitSkipCommit") + syncPackageLevelDetails := config.GetBoolWithFallback(cmd, "sync-package-details", "sync.syncPackageDetails") + target := config.GetStringWithFallback(cmd, "target", "sync.target") serviceDetails := api.GetServiceDetails(cmd) // Initialise HTTP executer diff --git a/internal/config/config.go b/internal/config/config.go index 1746b32..293efec 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -77,3 +77,98 @@ func verifyNoSensitiveContent(input string) (bool, error) { return true, nil } + +// GetStringWithFallback reads a string value from command flag, +// falling back to a nested config key if the flag wasn't explicitly set +func GetStringWithFallback(cmd *cobra.Command, flagName, configKey string) string { + // Check if flag was explicitly set on command line + if cmd.Flags().Changed(flagName) { + return GetString(cmd, flagName) + } + + // Try to get from nested config key + if viper.IsSet(configKey) { + return viper.GetString(configKey) + } + + // Fall back to flag default + return GetString(cmd, flagName) +} + +// GetBoolWithFallback reads a bool value from command flag, +// falling back to a nested config key if the flag wasn't explicitly set +func GetBoolWithFallback(cmd *cobra.Command, flagName, configKey string) bool { + // Check if flag was explicitly set on command line + if cmd.Flags().Changed(flagName) { + return GetBool(cmd, flagName) + } + + // Try to get from nested config key + if viper.IsSet(configKey) { + return viper.GetBool(configKey) + } + + // Fall back to flag default + return GetBool(cmd, flagName) +} + +// GetIntWithFallback reads an int value from command flag, +// falling back to a nested config key if the flag wasn't explicitly set +func GetIntWithFallback(cmd *cobra.Command, flagName, configKey string) int { + // Check if flag was explicitly set on command line + if cmd.Flags().Changed(flagName) { + return GetInt(cmd, flagName) + } + + // Try to get from nested config key + if viper.IsSet(configKey) { + return viper.GetInt(configKey) + } + + // Fall back to flag default + return GetInt(cmd, flagName) +} + +// GetStringSliceWithFallback reads a string slice value from command flag, +// falling back to a nested config key if the flag wasn't explicitly set +func GetStringSliceWithFallback(cmd *cobra.Command, flagName, configKey string) []string { + // Check if flag was explicitly set on command line + if cmd.Flags().Changed(flagName) { + return GetStringSlice(cmd, flagName) + } + + // Try to get from nested config key + if viper.IsSet(configKey) { + return viper.GetStringSlice(configKey) + } + + // Fall back to flag default + return GetStringSlice(cmd, flagName) +} + +// GetStringWithEnvExpandAndFallback reads a string value with environment variable expansion, +// falling back to a nested config key if the flag wasn't explicitly set +func GetStringWithEnvExpandAndFallback(cmd *cobra.Command, flagName, configKey string) (string, error) { + var val string + + // Check if flag was explicitly set on command line + if cmd.Flags().Changed(flagName) { + val = GetString(cmd, flagName) + } else if viper.IsSet(configKey) { + // Try to get from nested config key + val = viper.GetString(configKey) + } else { + // Fall back to flag default + val = GetString(cmd, flagName) + } + + // Expand environment variables + val = os.ExpandEnv(val) + + isNoSensContFound, err := verifyNoSensitiveContent(val) + if !isNoSensContFound { + return "", fmt.Errorf("Sensitive content found in flag %v: %w", flagName, err) + } + + return val, nil +} diff --git a/internal/deploy/config_loader.go b/internal/deploy/config_loader.go new file mode 100644 index 0000000..2c95774 --- /dev/null +++ b/internal/deploy/config_loader.go @@ -0,0 +1,390 @@ +package deploy + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/engswee/flashpipe/internal/models" + "gopkg.in/yaml.v3" +) + +// ConfigSource represents the type of configuration source +type ConfigSource string + +const ( + SourceFile ConfigSource = "file" + SourceFolder ConfigSource = "folder" + SourceURL ConfigSource = "url" +) + +// ConfigLoader handles loading deployment configurations from various sources +type ConfigLoader struct { + Source ConfigSource + Path string + URL string + AuthToken string + AuthType string // "bearer" or "basic" + Username string // for basic auth + Password string // for basic auth + FilePattern string // pattern for config files in folders + Debug bool +} + +// DeployConfigFile represents a loaded config file with metadata +type DeployConfigFile struct { + Config *models.DeployConfig + Source string // original source path/URL + FileName string // base filename + Order int // processing order +} + +// NewConfigLoader creates a new config loader +func NewConfigLoader() *ConfigLoader { + return &ConfigLoader{ + Source: SourceFile, + FilePattern: "*.y*ml", // default pattern matches .yml and .yaml + AuthType: "bearer", + } +} + +// DetectSource automatically detects the source type based on the path +func (cl *ConfigLoader) DetectSource(path string) error { + // Check if it's a URL + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + cl.Source = SourceURL + cl.URL = path + return nil + } + + // Check if path exists + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("path does not exist: %s", path) + } + + // Determine if it's a file or directory + if info.IsDir() { + cl.Source = SourceFolder + cl.Path = path + } else { + cl.Source = SourceFile + cl.Path = path + } + + return nil +} + +// LoadConfigs loads all configuration files based on the source type +func (cl *ConfigLoader) LoadConfigs() ([]*DeployConfigFile, error) { + switch cl.Source { + case SourceFile: + return cl.loadSingleFile() + case SourceFolder: + return cl.loadFolder() + case SourceURL: + return cl.loadURL() + default: + return nil, fmt.Errorf("unsupported source type: %s", cl.Source) + } +} + +// loadSingleFile loads a single configuration file +func (cl *ConfigLoader) loadSingleFile() ([]*DeployConfigFile, error) { + var config models.DeployConfig + if err := readYAML(cl.Path, &config); err != nil { + return nil, fmt.Errorf("failed to load config file %s: %w", cl.Path, err) + } + + return []*DeployConfigFile{ + { + Config: &config, + Source: cl.Path, + FileName: filepath.Base(cl.Path), + Order: 0, + }, + }, nil +} + +// loadFolder loads all matching configuration files from a folder (including subdirectories recursively) +func (cl *ConfigLoader) loadFolder() ([]*DeployConfigFile, error) { + var configFiles []*DeployConfigFile + var files []string + + if cl.Debug { + fmt.Printf("Scanning directory recursively: %s\n", cl.Path) + fmt.Printf("File pattern: %s\n", cl.FilePattern) + } + + // Walk through directory and all subdirectories recursively + err := filepath.Walk(cl.Path, func(path string, info os.FileInfo, err error) error { + if err != nil { + // Log error but continue walking + if cl.Debug { + fmt.Printf("Warning: Error accessing path %s: %v\n", path, err) + } + return nil // Continue walking despite errors + } + + // Skip directories (but continue walking into them) + if info.IsDir() { + if cl.Debug && path != cl.Path { + fmt.Printf("Entering subdirectory: %s\n", path) + } + return nil + } + + // Check if file matches pattern + matched, err := filepath.Match(cl.FilePattern, filepath.Base(path)) + if err != nil { + return fmt.Errorf("invalid file pattern: %w", err) + } + + if matched { + // Get relative path for better display + relPath, _ := filepath.Rel(cl.Path, path) + if cl.Debug { + fmt.Printf("Found matching file: %s\n", relPath) + } + files = append(files, path) + } + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to walk directory: %w", err) + } + + if len(files) == 0 { + return nil, fmt.Errorf("no config files found matching pattern '%s' in %s (searched recursively)", cl.FilePattern, cl.Path) + } + + if cl.Debug { + fmt.Printf("Found %d matching file(s)\n", len(files)) + } + + // Sort files alphabetically for consistent processing order + sort.Strings(files) + + if cl.Debug { + fmt.Println("Processing files in alphabetical order:") + for i, f := range files { + relPath, _ := filepath.Rel(cl.Path, f) + fmt.Printf(" %d. %s\n", i+1, relPath) + } + } + + // Load each file + successCount := 0 + for i, filePath := range files { + var config models.DeployConfig + if err := readYAML(filePath, &config); err != nil { + relPath, _ := filepath.Rel(cl.Path, filePath) + if cl.Debug { + fmt.Printf("Warning: Failed to load config file %s: %v\n", relPath, err) + } + continue + } + + // Get relative path from base directory for better display + relPath, _ := filepath.Rel(cl.Path, filePath) + + configFiles = append(configFiles, &DeployConfigFile{ + Config: &config, + Source: filePath, + FileName: relPath, + Order: i, + }) + + successCount++ + if cl.Debug { + fmt.Printf("✓ Loaded config file: %s (order: %d)\n", relPath, i) + } + } + + if len(configFiles) == 0 { + return nil, fmt.Errorf("no valid config files found in %s (found %d file(s) but all failed to parse)", cl.Path, len(files)) + } + + if cl.Debug { + fmt.Printf("\nSuccessfully loaded %d config file(s) out of %d found\n", successCount, len(files)) + } + + return configFiles, nil +} + +// loadURL loads a configuration file from a remote URL +func (cl *ConfigLoader) loadURL() ([]*DeployConfigFile, error) { + if cl.Debug { + fmt.Printf("Fetching config from URL: %s\n", cl.URL) + } + + // Create HTTP client + client := &http.Client{} + + // Create request + req, err := http.NewRequest("GET", cl.URL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // Add authentication if provided + if cl.AuthToken != "" { + if cl.AuthType == "bearer" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cl.AuthToken)) + if cl.Debug { + fmt.Println("Using Bearer token authentication") + } + } else if cl.AuthType == "basic" { + req.SetBasicAuth(cl.Username, cl.Password) + if cl.Debug { + fmt.Printf("Using Basic authentication with username: %s\n", cl.Username) + } + } + } else if cl.Username != "" && cl.Password != "" { + // Use basic auth if username/password provided without token + req.SetBasicAuth(cl.Username, cl.Password) + if cl.Debug { + fmt.Printf("Using Basic authentication with username: %s\n", cl.Username) + } + } + + // Make request + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch URL: %w", err) + } + defer resp.Body.Close() + + // Check response status + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to fetch URL: status %d", resp.StatusCode) + } + + if cl.Debug { + fmt.Printf("Successfully fetched config (status: %d)\n", resp.StatusCode) + } + + // Read response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + + // Save to temporary file for YAML parsing + tempFile, err := os.CreateTemp("", "deploy-config-*.yml") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + defer os.Remove(tempFile.Name()) + + if _, err := tempFile.Write(body); err != nil { + return nil, fmt.Errorf("failed to write temp file: %w", err) + } + tempFile.Close() + + // Parse YAML + var config models.DeployConfig + if err := readYAML(tempFile.Name(), &config); err != nil { + return nil, fmt.Errorf("failed to parse config from URL: %w", err) + } + + // Extract filename from URL + urlParts := strings.Split(cl.URL, "/") + fileName := urlParts[len(urlParts)-1] + if fileName == "" { + fileName = "remote-config.yml" + } + + if cl.Debug { + fmt.Printf("✓ Successfully parsed config from URL\n") + } + + return []*DeployConfigFile{ + { + Config: &config, + Source: cl.URL, + FileName: fileName, + Order: 0, + }, + }, nil +} + +// MergeConfigs merges multiple deployment configs into a single config +func MergeConfigs(configs []*DeployConfigFile) (*models.DeployConfig, error) { + if len(configs) == 0 { + return nil, fmt.Errorf("no configs to merge") + } + + // Merged config has NO deployment prefix since each package will have its own + merged := &models.DeployConfig{ + DeploymentPrefix: "", + Packages: []models.Package{}, + } + + // Track fully qualified package IDs (with prefix) to detect true duplicates + packageMap := make(map[string]string) // map[fullyQualifiedID]sourceFile + + // Merge packages from all configs + for _, configFile := range configs { + configPrefix := configFile.Config.DeploymentPrefix + + for _, pkg := range configFile.Config.Packages { + // Create a copy of the package to avoid modifying the original + mergedPkg := pkg + + // Calculate the fully qualified package ID + fullyQualifiedID := pkg.ID + if configPrefix != "" { + fullyQualifiedID = configPrefix + "" + pkg.ID + + // Update the package ID and display name with prefix + mergedPkg.ID = fullyQualifiedID + + // Update display name if it exists + if mergedPkg.DisplayName != "" { + mergedPkg.DisplayName = configPrefix + " - " + mergedPkg.DisplayName + } else { + mergedPkg.DisplayName = configPrefix + " - " + pkg.ID + } + } + + // Check for duplicate fully qualified IDs + if existingSource, exists := packageMap[fullyQualifiedID]; exists { + return nil, fmt.Errorf("duplicate package ID '%s' found in %s (already exists from %s)", + fullyQualifiedID, configFile.FileName, existingSource) + } + + // Apply prefix to all artifact IDs as well + if configPrefix != "" { + for i := range mergedPkg.Artifacts { + mergedPkg.Artifacts[i].Id = configPrefix + "_" + mergedPkg.Artifacts[i].Id + } + } + + packageMap[fullyQualifiedID] = configFile.FileName + merged.Packages = append(merged.Packages, mergedPkg) + } + } + + return merged, nil +} + +// readYAML reads and unmarshals a YAML file +func readYAML(path string, v interface{}) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + if err := yaml.Unmarshal(data, v); err != nil { + return fmt.Errorf("failed to parse YAML: %w", err) + } + + return nil +} diff --git a/internal/deploy/config_loader_test.go b/internal/deploy/config_loader_test.go new file mode 100644 index 0000000..cd775aa --- /dev/null +++ b/internal/deploy/config_loader_test.go @@ -0,0 +1,558 @@ +package deploy + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/engswee/flashpipe/internal/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewConfigLoader(t *testing.T) { + loader := NewConfigLoader() + assert.NotNil(t, loader) + assert.Equal(t, SourceFile, loader.Source) + assert.Equal(t, "*.y*ml", loader.FilePattern) + assert.Equal(t, "bearer", loader.AuthType) +} + +func TestDetectSource_File(t *testing.T) { + tempDir, err := os.MkdirTemp("", "config-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create a test file + testFile := filepath.Join(tempDir, "config.yml") + err = os.WriteFile(testFile, []byte("test"), 0644) + require.NoError(t, err) + + loader := NewConfigLoader() + err = loader.DetectSource(testFile) + require.NoError(t, err) + + assert.Equal(t, SourceFile, loader.Source) + assert.Equal(t, testFile, loader.Path) +} + +func TestDetectSource_Folder(t *testing.T) { + tempDir, err := os.MkdirTemp("", "config-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + loader := NewConfigLoader() + err = loader.DetectSource(tempDir) + require.NoError(t, err) + + assert.Equal(t, SourceFolder, loader.Source) + assert.Equal(t, tempDir, loader.Path) +} + +func TestDetectSource_URL(t *testing.T) { + loader := NewConfigLoader() + + tests := []struct { + name string + url string + }{ + {"http", "http://example.com/config.yml"}, + {"https", "https://example.com/config.yml"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := loader.DetectSource(tt.url) + require.NoError(t, err) + + assert.Equal(t, SourceURL, loader.Source) + assert.Equal(t, tt.url, loader.URL) + }) + } +} + +func TestDetectSource_NonExistent(t *testing.T) { + loader := NewConfigLoader() + err := loader.DetectSource("/nonexistent/path") + assert.Error(t, err) + assert.Contains(t, err.Error(), "path does not exist") +} + +func TestLoadSingleFile(t *testing.T) { + tempDir, err := os.MkdirTemp("", "config-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create test config + configFile := filepath.Join(tempDir, "test-config.yml") + configContent := ` +deploymentPrefix: TEST +packages: + - integrationSuiteId: Package1 + displayName: Test Package 1 + artifacts: + - artifactId: artifact1 + displayName: Artifact 1 + type: Integration +` + err = os.WriteFile(configFile, []byte(configContent), 0644) + require.NoError(t, err) + + loader := NewConfigLoader() + loader.Path = configFile + loader.Source = SourceFile + + configs, err := loader.LoadConfigs() + require.NoError(t, err) + require.Len(t, configs, 1) + + assert.Equal(t, "TEST", configs[0].Config.DeploymentPrefix) + assert.Len(t, configs[0].Config.Packages, 1) + assert.Equal(t, "Package1", configs[0].Config.Packages[0].ID) + assert.Equal(t, configFile, configs[0].Source) + assert.Equal(t, "test-config.yml", configs[0].FileName) +} + +func TestLoadFolder_SingleFile(t *testing.T) { + tempDir, err := os.MkdirTemp("", "config-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create test config + configFile := filepath.Join(tempDir, "config.yml") + configContent := ` +deploymentPrefix: TEST +packages: + - integrationSuiteId: Package1 +` + err = os.WriteFile(configFile, []byte(configContent), 0644) + require.NoError(t, err) + + loader := NewConfigLoader() + loader.Path = tempDir + loader.Source = SourceFolder + + configs, err := loader.LoadConfigs() + require.NoError(t, err) + require.Len(t, configs, 1) + + assert.Equal(t, "TEST", configs[0].Config.DeploymentPrefix) +} + +func TestLoadFolder_MultipleFiles(t *testing.T) { + tempDir, err := os.MkdirTemp("", "config-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create multiple test configs + configs := map[string]string{ + "a-config.yml": ` +deploymentPrefix: A +packages: + - integrationSuiteId: PackageA +`, + "b-config.yaml": ` +deploymentPrefix: B +packages: + - integrationSuiteId: PackageB +`, + "c-config.yml": ` +deploymentPrefix: C +packages: + - integrationSuiteId: PackageC +`, + } + + for filename, content := range configs { + err = os.WriteFile(filepath.Join(tempDir, filename), []byte(content), 0644) + require.NoError(t, err) + } + + loader := NewConfigLoader() + loader.Path = tempDir + loader.Source = SourceFolder + + loadedConfigs, err := loader.LoadConfigs() + require.NoError(t, err) + require.Len(t, loadedConfigs, 3) + + // Verify alphabetical order + assert.Equal(t, "a-config.yml", loadedConfigs[0].FileName) + assert.Equal(t, "b-config.yaml", loadedConfigs[1].FileName) + assert.Equal(t, "c-config.yml", loadedConfigs[2].FileName) + + // Verify order numbers + assert.Equal(t, 0, loadedConfigs[0].Order) + assert.Equal(t, 1, loadedConfigs[1].Order) + assert.Equal(t, 2, loadedConfigs[2].Order) +} + +func TestLoadFolder_Recursive(t *testing.T) { + tempDir, err := os.MkdirTemp("", "config-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create subdirectory structure + subDir1 := filepath.Join(tempDir, "env1") + subDir2 := filepath.Join(tempDir, "env2", "configs") + err = os.MkdirAll(subDir1, 0755) + require.NoError(t, err) + err = os.MkdirAll(subDir2, 0755) + require.NoError(t, err) + + // Create configs in different directories + configs := map[string]string{ + filepath.Join(tempDir, "root.yml"): "deploymentPrefix: ROOT\npackages: []", + filepath.Join(subDir1, "env1.yml"): "deploymentPrefix: ENV1\npackages: []", + filepath.Join(subDir2, "deep.yml"): "deploymentPrefix: DEEP\npackages: []", + } + + for path, content := range configs { + err = os.WriteFile(path, []byte(content), 0644) + require.NoError(t, err) + } + + loader := NewConfigLoader() + loader.Path = tempDir + loader.Source = SourceFolder + + loadedConfigs, err := loader.LoadConfigs() + require.NoError(t, err) + require.Len(t, loadedConfigs, 3) + + // Verify all files were found (alphabetically sorted by full path) + foundPrefixes := make(map[string]bool) + for _, cfg := range loadedConfigs { + foundPrefixes[cfg.Config.DeploymentPrefix] = true + } + + assert.True(t, foundPrefixes["ROOT"]) + assert.True(t, foundPrefixes["ENV1"]) + assert.True(t, foundPrefixes["DEEP"]) +} + +func TestLoadFolder_NoMatches(t *testing.T) { + tempDir, err := os.MkdirTemp("", "config-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create non-matching file + err = os.WriteFile(filepath.Join(tempDir, "test.txt"), []byte("test"), 0644) + require.NoError(t, err) + + loader := NewConfigLoader() + loader.Path = tempDir + loader.Source = SourceFolder + + _, err = loader.LoadConfigs() + assert.Error(t, err) + assert.Contains(t, err.Error(), "no config files found") +} + +func TestLoadFolder_CustomPattern(t *testing.T) { + tempDir, err := os.MkdirTemp("", "config-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create files with different extensions + err = os.WriteFile(filepath.Join(tempDir, "config.yml"), []byte("deploymentPrefix: YML\npackages: []"), 0644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tempDir, "config.json"), []byte("deploymentPrefix: JSON\npackages: []"), 0644) + require.NoError(t, err) + + loader := NewConfigLoader() + loader.Path = tempDir + loader.Source = SourceFolder + loader.FilePattern = "*.json" + + configs, err := loader.LoadConfigs() + require.NoError(t, err) + // Should only load the .json file (not .yml) + require.Len(t, configs, 1) + assert.Equal(t, "JSON", configs[0].Config.DeploymentPrefix) +} + +func TestLoadFolder_InvalidYAML(t *testing.T) { + tempDir, err := os.MkdirTemp("", "config-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create invalid YAML file + err = os.WriteFile(filepath.Join(tempDir, "invalid.yml"), []byte("invalid: yaml: content:"), 0644) + require.NoError(t, err) + + // Create valid YAML file + err = os.WriteFile(filepath.Join(tempDir, "valid.yml"), []byte("deploymentPrefix: VALID\npackages: []"), 0644) + require.NoError(t, err) + + loader := NewConfigLoader() + loader.Path = tempDir + loader.Source = SourceFolder + + configs, err := loader.LoadConfigs() + require.NoError(t, err) + // Should only load the valid file + require.Len(t, configs, 1) + assert.Equal(t, "VALID", configs[0].Config.DeploymentPrefix) +} + +func TestLoadURL_Success(t *testing.T) { + configContent := ` +deploymentPrefix: REMOTE +packages: + - integrationSuiteId: RemotePackage +` + + // Create test server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(configContent)) + })) + defer server.Close() + + loader := NewConfigLoader() + loader.URL = server.URL + loader.Source = SourceURL + + configs, err := loader.LoadConfigs() + require.NoError(t, err) + require.Len(t, configs, 1) + + assert.Equal(t, "REMOTE", configs[0].Config.DeploymentPrefix) + assert.Equal(t, server.URL, configs[0].Source) +} + +func TestLoadURL_WithBearerAuth(t *testing.T) { + expectedToken := "test-token-123" + configContent := "deploymentPrefix: AUTH\npackages: []" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if auth != "Bearer "+expectedToken { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.WriteHeader(http.StatusOK) + w.Write([]byte(configContent)) + })) + defer server.Close() + + loader := NewConfigLoader() + loader.URL = server.URL + loader.Source = SourceURL + loader.AuthToken = expectedToken + loader.AuthType = "bearer" + + configs, err := loader.LoadConfigs() + require.NoError(t, err) + require.Len(t, configs, 1) + assert.Equal(t, "AUTH", configs[0].Config.DeploymentPrefix) +} + +func TestLoadURL_WithBasicAuth(t *testing.T) { + expectedUser := "testuser" + expectedPass := "testpass" + configContent := "deploymentPrefix: BASIC\npackages: []" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + user, pass, ok := r.BasicAuth() + if !ok || user != expectedUser || pass != expectedPass { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.WriteHeader(http.StatusOK) + w.Write([]byte(configContent)) + })) + defer server.Close() + + loader := NewConfigLoader() + loader.URL = server.URL + loader.Source = SourceURL + loader.Username = expectedUser + loader.Password = expectedPass + + configs, err := loader.LoadConfigs() + require.NoError(t, err) + require.Len(t, configs, 1) + assert.Equal(t, "BASIC", configs[0].Config.DeploymentPrefix) +} + +func TestLoadURL_HTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + loader := NewConfigLoader() + loader.URL = server.URL + loader.Source = SourceURL + + _, err := loader.LoadConfigs() + assert.Error(t, err) + assert.Contains(t, err.Error(), "status 404") +} + +func TestMergeConfigs_Single(t *testing.T) { + configs := []*DeployConfigFile{ + { + Config: &models.DeployConfig{ + DeploymentPrefix: "TEST", + Packages: []models.Package{ + {ID: "Package1"}, + }, + }, + FileName: "test.yml", + }, + } + + merged, err := MergeConfigs(configs) + require.NoError(t, err) + + assert.Equal(t, "", merged.DeploymentPrefix) + assert.Len(t, merged.Packages, 1) + assert.Equal(t, "TESTPackage1", merged.Packages[0].ID) +} + +func TestMergeConfigs_Multiple(t *testing.T) { + configs := []*DeployConfigFile{ + { + Config: &models.DeployConfig{ + DeploymentPrefix: "DEV", + Packages: []models.Package{ + {ID: "Package1", DisplayName: "Pkg 1"}, + }, + }, + FileName: "dev.yml", + }, + { + Config: &models.DeployConfig{ + DeploymentPrefix: "QA", + Packages: []models.Package{ + {ID: "Package2", DisplayName: "Pkg 2"}, + }, + }, + FileName: "qa.yml", + }, + } + + merged, err := MergeConfigs(configs) + require.NoError(t, err) + + assert.Equal(t, "", merged.DeploymentPrefix) + assert.Len(t, merged.Packages, 2) + + // Verify prefixes applied + assert.Equal(t, "DEVPackage1", merged.Packages[0].ID) + assert.Equal(t, "DEV - Pkg 1", merged.Packages[0].DisplayName) + + assert.Equal(t, "QAPackage2", merged.Packages[1].ID) + assert.Equal(t, "QA - Pkg 2", merged.Packages[1].DisplayName) +} + +func TestMergeConfigs_NoPrefix(t *testing.T) { + configs := []*DeployConfigFile{ + { + Config: &models.DeployConfig{ + DeploymentPrefix: "", + Packages: []models.Package{ + {ID: "Package1"}, + }, + }, + FileName: "test.yml", + }, + } + + merged, err := MergeConfigs(configs) + require.NoError(t, err) + + assert.Len(t, merged.Packages, 1) + assert.Equal(t, "Package1", merged.Packages[0].ID) // No prefix applied +} + +func TestMergeConfigs_DuplicateID(t *testing.T) { + configs := []*DeployConfigFile{ + { + Config: &models.DeployConfig{ + DeploymentPrefix: "ENV", + Packages: []models.Package{ + {ID: "Package1"}, + }, + }, + FileName: "config1.yml", + }, + { + Config: &models.DeployConfig{ + DeploymentPrefix: "ENV", + Packages: []models.Package{ + {ID: "Package1"}, // Same fully qualified ID + }, + }, + FileName: "config2.yml", + }, + } + + _, err := MergeConfigs(configs) + assert.Error(t, err) + assert.Contains(t, err.Error(), "duplicate package ID") + assert.Contains(t, err.Error(), "ENVPackage1") +} + +func TestMergeConfigs_ArtifactPrefixing(t *testing.T) { + configs := []*DeployConfigFile{ + { + Config: &models.DeployConfig{ + DeploymentPrefix: "TEST", + Packages: []models.Package{ + { + ID: "Package1", + Artifacts: []models.Artifact{ + {Id: "artifact1", Type: "Integration"}, + {Id: "artifact2", Type: "Integration"}, + }, + }, + }, + }, + FileName: "test.yml", + }, + } + + merged, err := MergeConfigs(configs) + require.NoError(t, err) + + require.Len(t, merged.Packages, 1) + require.Len(t, merged.Packages[0].Artifacts, 2) + + // Verify artifact IDs are prefixed + assert.Equal(t, "TEST_artifact1", merged.Packages[0].Artifacts[0].Id) + assert.Equal(t, "TEST_artifact2", merged.Packages[0].Artifacts[1].Id) +} + +func TestMergeConfigs_Empty(t *testing.T) { + configs := []*DeployConfigFile{} + + _, err := MergeConfigs(configs) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no configs to merge") +} + +func TestMergeConfigs_DisplayNameGeneration(t *testing.T) { + configs := []*DeployConfigFile{ + { + Config: &models.DeployConfig{ + DeploymentPrefix: "PREFIX", + Packages: []models.Package{ + {ID: "Package1"}, // No display name + }, + }, + FileName: "test.yml", + }, + } + + merged, err := MergeConfigs(configs) + require.NoError(t, err) + + // Display name should be generated from prefix and ID + assert.Equal(t, "PREFIX - Package1", merged.Packages[0].DisplayName) +} diff --git a/internal/deploy/utils.go b/internal/deploy/utils.go new file mode 100644 index 0000000..88da701 --- /dev/null +++ b/internal/deploy/utils.go @@ -0,0 +1,281 @@ +package deploy + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// FileExists checks if a file exists +func FileExists(path string) bool { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return false + } + return err == nil && !info.IsDir() +} + +// DirExists checks if a directory exists +func DirExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +// ValidateDeploymentPrefix validates that the deployment prefix only contains allowed characters +func ValidateDeploymentPrefix(prefix string) error { + if prefix == "" { + return nil // Empty prefix is valid + } + + // Only allow alphanumeric and underscores + matched, err := regexp.MatchString("^[a-zA-Z0-9_]+$", prefix) + if err != nil { + return fmt.Errorf("regex error: %w", err) + } + + if !matched { + return fmt.Errorf("deployment prefix can only contain alphanumeric characters (a-z, A-Z, 0-9) and underscores (_)") + } + + return nil +} + +// CopyDir recursively copies a directory +func CopyDir(src, dst string) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Get relative path + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + + targetPath := filepath.Join(dst, relPath) + + if info.IsDir() { + return os.MkdirAll(targetPath, info.Mode()) + } + + // Copy file + data, err := os.ReadFile(path) + if err != nil { + return err + } + + return os.WriteFile(targetPath, data, info.Mode()) + }) +} + +// UpdateManifestBundleName updates the Bundle-Name and Bundle-SymbolicName in MANIFEST.MF +func UpdateManifestBundleName(manifestPath, bundleSymbolicName, bundleName, outputPath string) error { + data, err := os.ReadFile(manifestPath) + if err != nil { + return fmt.Errorf("failed to read MANIFEST.MF: %w", err) + } + + // Detect line ending style (CRLF or LF) + lineEnding := "\n" + if strings.Contains(string(data), "\r\n") { + lineEnding = "\r\n" + } + + // Split lines + content := string(data) + lines := strings.Split(content, lineEnding) + + var result []string + bundleNameFound := false + bundleSymbolicNameFound := false + + for _, line := range lines { + trimmedLower := strings.ToLower(strings.TrimSpace(line)) + + if strings.HasPrefix(trimmedLower, "bundle-name:") { + result = append(result, fmt.Sprintf("Bundle-Name: %s", bundleName)) + bundleNameFound = true + } else if strings.HasPrefix(trimmedLower, "bundle-symbolicname:") { + result = append(result, fmt.Sprintf("Bundle-SymbolicName: %s", bundleSymbolicName)) + bundleSymbolicNameFound = true + } else { + result = append(result, line) + } + } + + // Add Bundle-Name if not found + if !bundleNameFound { + result = append(result, fmt.Sprintf("Bundle-Name: %s", bundleName)) + } + + // Add Bundle-SymbolicName if not found + if !bundleSymbolicNameFound { + result = append(result, fmt.Sprintf("Bundle-SymbolicName: %s", bundleSymbolicName)) + } + + // Write to output path with original line endings and ensure final newline + finalContent := strings.Join(result, lineEnding) + if !strings.HasSuffix(finalContent, lineEnding) { + finalContent += lineEnding + } + + // Create directory if needed + if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + if err := os.WriteFile(outputPath, []byte(finalContent), 0644); err != nil { + return fmt.Errorf("failed to write MANIFEST.MF: %w", err) + } + + return nil +} + +// MergeParametersFile reads parameters.prop, applies overrides, and writes to outputPath +func MergeParametersFile(paramsPath string, overrides map[string]interface{}, outputPath string) error { + var lineEnding string = "\n" + params := make(map[string]string) + paramKeys := []string{} // Track order of keys + + // Read existing file if it exists + if FileExists(paramsPath) { + data, err := os.ReadFile(paramsPath) + if err != nil { + return fmt.Errorf("failed to read parameters.prop: %w", err) + } + + // Detect line ending style + content := string(data) + if strings.Contains(content, "\r\n") { + lineEnding = "\r\n" + } + + // Split and process lines + lines := strings.Split(content, lineEnding) + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Keep comments and empty lines as-is + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + // Parse key=value + parts := strings.SplitN(trimmed, "=", 2) + if len(parts) == 2 { + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + params[key] = value + paramKeys = append(paramKeys, key) + } + } + } + + // Apply overrides + for key, value := range overrides { + valStr := fmt.Sprintf("%v", value) + if _, exists := params[key]; !exists { + // New key, add to order + paramKeys = append(paramKeys, key) + } + params[key] = valStr + } + + // Write back with preserved order + var result []string + for _, key := range paramKeys { + result = append(result, fmt.Sprintf("%s=%s", key, params[key])) + } + + // Join with original line endings and ensure final newline + finalContent := strings.Join(result, lineEnding) + if !strings.HasSuffix(finalContent, lineEnding) { + finalContent += lineEnding + } + + // Create directory if needed + if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + if err := os.WriteFile(outputPath, []byte(finalContent), 0644); err != nil { + return fmt.Errorf("failed to write parameters.prop: %w", err) + } + + return nil +} + +// FindParametersFile finds parameters.prop in various possible locations +func FindParametersFile(artifactDir string) string { + possiblePaths := []string{ + filepath.Join(artifactDir, "src", "main", "resources", "parameters.prop"), + filepath.Join(artifactDir, "src", "main", "resources", "script", "parameters.prop"), + filepath.Join(artifactDir, "parameters.prop"), + } + + for _, path := range possiblePaths { + if FileExists(path) { + return path + } + } + + // Return default path even if it doesn't exist + return possiblePaths[0] +} + +// GetManifestHeaders reads headers from MANIFEST.MF file +func GetManifestHeaders(manifestPath string) (map[string]string, error) { + metadata := make(map[string]string) + + if !FileExists(manifestPath) { + return metadata, nil + } + + file, err := os.Open(manifestPath) + if err != nil { + return nil, fmt.Errorf("failed to open MANIFEST.MF: %w", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + var currentKey string + var currentValue strings.Builder + + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + + if strings.Contains(trimmed, ":") && !strings.HasPrefix(line, " ") && !strings.HasPrefix(line, "\t") { + // New key-value pair + if currentKey != "" { + metadata[currentKey] = strings.TrimSpace(currentValue.String()) + } + parts := strings.SplitN(trimmed, ":", 2) + if len(parts) == 2 { + currentKey = strings.TrimSpace(parts[0]) + currentValue.Reset() + currentValue.WriteString(strings.TrimSpace(parts[1])) + } + } else if currentKey != "" && (strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t")) { + // Continuation line + currentValue.WriteString(" ") + currentValue.WriteString(strings.TrimSpace(line)) + } + } + + // Add the last entry + if currentKey != "" { + metadata[currentKey] = strings.TrimSpace(currentValue.String()) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to read MANIFEST.MF: %w", err) + } + + return metadata, nil +} diff --git a/internal/deploy/utils_test.go b/internal/deploy/utils_test.go new file mode 100644 index 0000000..bfbf228 --- /dev/null +++ b/internal/deploy/utils_test.go @@ -0,0 +1,562 @@ +package deploy + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFileExists(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create a file + testFile := filepath.Join(tempDir, "test.txt") + err = os.WriteFile(testFile, []byte("test"), 0644) + require.NoError(t, err) + + // Create a directory + testDir := filepath.Join(tempDir, "testdir") + err = os.MkdirAll(testDir, 0755) + require.NoError(t, err) + + tests := []struct { + name string + path string + want bool + }{ + {"existing file", testFile, true}, + {"directory (not a file)", testDir, false}, + {"non-existent", filepath.Join(tempDir, "nonexistent.txt"), false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := FileExists(tt.path) + assert.Equal(t, tt.want, result) + }) + } +} + +func TestDirExists(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create a file + testFile := filepath.Join(tempDir, "test.txt") + err = os.WriteFile(testFile, []byte("test"), 0644) + require.NoError(t, err) + + // Create a directory + testDir := filepath.Join(tempDir, "testdir") + err = os.MkdirAll(testDir, 0755) + require.NoError(t, err) + + tests := []struct { + name string + path string + want bool + }{ + {"existing directory", testDir, true}, + {"file (not a directory)", testFile, false}, + {"non-existent", filepath.Join(tempDir, "nonexistent"), false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := DirExists(tt.path) + assert.Equal(t, tt.want, result) + }) + } +} + +func TestValidateDeploymentPrefix_Valid(t *testing.T) { + tests := []struct { + name string + prefix string + }{ + {"empty prefix", ""}, + {"alphanumeric", "Test123"}, + {"uppercase", "PRODUCTION"}, + {"lowercase", "development"}, + {"with underscores", "dev_environment_1"}, + {"numbers only", "123"}, + {"letters only", "abc"}, + {"single char", "A"}, + {"underscore only", "_"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateDeploymentPrefix(tt.prefix) + assert.NoError(t, err) + }) + } +} + +func TestValidateDeploymentPrefix_Invalid(t *testing.T) { + tests := []struct { + name string + prefix string + }{ + {"with dash", "dev-env"}, + {"with space", "dev env"}, + {"with dot", "dev.env"}, + {"with special chars", "dev@env"}, + {"with slash", "dev/env"}, + {"with brackets", "dev[env]"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateDeploymentPrefix(tt.prefix) + assert.Error(t, err) + assert.Contains(t, err.Error(), "deployment prefix can only contain") + }) + } +} + +func TestCopyDir(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create source directory structure + srcDir := filepath.Join(tempDir, "src") + err = os.MkdirAll(filepath.Join(srcDir, "subdir"), 0755) + require.NoError(t, err) + + // Create files + err = os.WriteFile(filepath.Join(srcDir, "file1.txt"), []byte("content1"), 0644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(srcDir, "subdir", "file2.txt"), []byte("content2"), 0644) + require.NoError(t, err) + + // Copy directory + dstDir := filepath.Join(tempDir, "dst") + err = CopyDir(srcDir, dstDir) + require.NoError(t, err) + + // Verify copied files + content1, err := os.ReadFile(filepath.Join(dstDir, "file1.txt")) + require.NoError(t, err) + assert.Equal(t, "content1", string(content1)) + + content2, err := os.ReadFile(filepath.Join(dstDir, "subdir", "file2.txt")) + require.NoError(t, err) + assert.Equal(t, "content2", string(content2)) + + // Verify directory exists + assert.True(t, DirExists(filepath.Join(dstDir, "subdir"))) +} + +func TestUpdateManifestBundleName_BothFieldsExist(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + manifestContent := `Manifest-Version: 1.0 +Bundle-Name: OldName +Bundle-SymbolicName: OldSymbolicName +Bundle-Version: 1.0.0 +` + manifestPath := filepath.Join(tempDir, "MANIFEST.MF") + err = os.WriteFile(manifestPath, []byte(manifestContent), 0644) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "MANIFEST_OUT.MF") + err = UpdateManifestBundleName(manifestPath, "NewSymbolicName", "NewName", outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "Bundle-Name: NewName") + assert.Contains(t, contentStr, "Bundle-SymbolicName: NewSymbolicName") + assert.NotContains(t, contentStr, "OldName") + assert.NotContains(t, contentStr, "OldSymbolicName") + assert.Contains(t, contentStr, "Bundle-Version: 1.0.0") +} + +func TestUpdateManifestBundleName_FieldsMissing(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + manifestContent := `Manifest-Version: 1.0 +Bundle-Version: 1.0.0 +` + manifestPath := filepath.Join(tempDir, "MANIFEST.MF") + err = os.WriteFile(manifestPath, []byte(manifestContent), 0644) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "MANIFEST_OUT.MF") + err = UpdateManifestBundleName(manifestPath, "NewSymbolicName", "NewName", outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "Bundle-Name: NewName") + assert.Contains(t, contentStr, "Bundle-SymbolicName: NewSymbolicName") +} + +func TestUpdateManifestBundleName_PreservesLineEndings_LF(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + manifestContent := "Manifest-Version: 1.0\nBundle-Name: OldName\n" + manifestPath := filepath.Join(tempDir, "MANIFEST.MF") + err = os.WriteFile(manifestPath, []byte(manifestContent), 0644) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "MANIFEST_OUT.MF") + err = UpdateManifestBundleName(manifestPath, "NewSymbolicName", "NewName", outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + // Should use LF + assert.Contains(t, string(content), "\n") + assert.NotContains(t, string(content), "\r\n") +} + +func TestUpdateManifestBundleName_PreservesLineEndings_CRLF(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + manifestContent := "Manifest-Version: 1.0\r\nBundle-Name: OldName\r\n" + manifestPath := filepath.Join(tempDir, "MANIFEST.MF") + err = os.WriteFile(manifestPath, []byte(manifestContent), 0644) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "MANIFEST_OUT.MF") + err = UpdateManifestBundleName(manifestPath, "NewSymbolicName", "NewName", outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + // Should preserve CRLF + assert.Contains(t, string(content), "\r\n") +} + +func TestUpdateManifestBundleName_CaseInsensitive(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Mix case headers + manifestContent := `bundle-name: OldName +BUNDLE-SYMBOLICNAME: OldSymbolicName +` + manifestPath := filepath.Join(tempDir, "MANIFEST.MF") + err = os.WriteFile(manifestPath, []byte(manifestContent), 0644) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "MANIFEST_OUT.MF") + err = UpdateManifestBundleName(manifestPath, "NewSymbolicName", "NewName", outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "Bundle-Name: NewName") + assert.Contains(t, contentStr, "Bundle-SymbolicName: NewSymbolicName") +} + +func TestMergeParametersFile_NewFile(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + paramsPath := filepath.Join(tempDir, "parameters.prop") + outputPath := filepath.Join(tempDir, "output.prop") + + overrides := map[string]interface{}{ + "param1": "value1", + "param2": 123, + "param3": true, + } + + err = MergeParametersFile(paramsPath, overrides, outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "param1=value1") + assert.Contains(t, contentStr, "param2=123") + assert.Contains(t, contentStr, "param3=true") +} + +func TestMergeParametersFile_ExistingFile(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create existing parameters file + existingContent := `param1=oldvalue1 +param2=oldvalue2 +param3=oldvalue3 +` + paramsPath := filepath.Join(tempDir, "parameters.prop") + err = os.WriteFile(paramsPath, []byte(existingContent), 0644) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "output.prop") + + overrides := map[string]interface{}{ + "param2": "newvalue2", + "param4": "newvalue4", + } + + err = MergeParametersFile(paramsPath, overrides, outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + contentStr := string(content) + assert.Contains(t, contentStr, "param1=oldvalue1") // Unchanged + assert.Contains(t, contentStr, "param2=newvalue2") // Overridden + assert.Contains(t, contentStr, "param3=oldvalue3") // Unchanged + assert.Contains(t, contentStr, "param4=newvalue4") // New +} + +func TestMergeParametersFile_PreservesOrder(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + existingContent := `aaa=value1 +zzz=value2 +mmm=value3 +` + paramsPath := filepath.Join(tempDir, "parameters.prop") + err = os.WriteFile(paramsPath, []byte(existingContent), 0644) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "output.prop") + + overrides := map[string]interface{}{ + "bbb": "newvalue", + } + + err = MergeParametersFile(paramsPath, overrides, outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + lines := strings.Split(string(content), "\n") + var paramLines []string + for _, line := range lines { + line = strings.TrimSpace(line) + if line != "" { + paramLines = append(paramLines, line) + } + } + + // Original order should be preserved, new param added at end + assert.Equal(t, "aaa=value1", paramLines[0]) + assert.Equal(t, "zzz=value2", paramLines[1]) + assert.Equal(t, "mmm=value3", paramLines[2]) + assert.Equal(t, "bbb=newvalue", paramLines[3]) +} + +func TestMergeParametersFile_PreservesLineEndings_LF(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + existingContent := "param1=value1\nparam2=value2\n" + paramsPath := filepath.Join(tempDir, "parameters.prop") + err = os.WriteFile(paramsPath, []byte(existingContent), 0644) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "output.prop") + + err = MergeParametersFile(paramsPath, map[string]interface{}{}, outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + assert.Contains(t, string(content), "\n") + assert.NotContains(t, string(content), "\r\n") +} + +func TestMergeParametersFile_PreservesLineEndings_CRLF(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + existingContent := "param1=value1\r\nparam2=value2\r\n" + paramsPath := filepath.Join(tempDir, "parameters.prop") + err = os.WriteFile(paramsPath, []byte(existingContent), 0644) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "output.prop") + + err = MergeParametersFile(paramsPath, map[string]interface{}{}, outputPath) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + assert.Contains(t, string(content), "\r\n") +} + +func TestFindParametersFile(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + tests := []struct { + name string + setupFunc func(string) error + expectedPath string + }{ + { + name: "in src/main/resources", + setupFunc: func(dir string) error { + path := filepath.Join(dir, "src", "main", "resources") + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + return os.WriteFile(filepath.Join(path, "parameters.prop"), []byte("test"), 0644) + }, + expectedPath: "src/main/resources/parameters.prop", + }, + { + name: "in src/main/resources/script", + setupFunc: func(dir string) error { + path := filepath.Join(dir, "src", "main", "resources", "script") + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + return os.WriteFile(filepath.Join(path, "parameters.prop"), []byte("test"), 0644) + }, + expectedPath: "src/main/resources/script/parameters.prop", + }, + { + name: "in root", + setupFunc: func(dir string) error { + return os.WriteFile(filepath.Join(dir, "parameters.prop"), []byte("test"), 0644) + }, + expectedPath: "parameters.prop", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testDir, err := os.MkdirTemp(tempDir, "find-test-*") + require.NoError(t, err) + defer os.RemoveAll(testDir) + + err = tt.setupFunc(testDir) + require.NoError(t, err) + + result := FindParametersFile(testDir) + expected := filepath.Join(testDir, filepath.FromSlash(tt.expectedPath)) + assert.Equal(t, expected, result) + assert.True(t, FileExists(result)) + }) + } +} + +func TestFindParametersFile_NotFound(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + result := FindParametersFile(tempDir) + // Should return default path even if it doesn't exist + expected := filepath.Join(tempDir, "src", "main", "resources", "parameters.prop") + assert.Equal(t, expected, result) +} + +func TestGetManifestHeaders(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + manifestContent := `Manifest-Version: 1.0 +Bundle-Name: Test Bundle +Bundle-SymbolicName: com.test.bundle +Bundle-Version: 1.0.0 +Import-Package: javax.xml.bind, + javax.xml.stream +Export-Package: com.test.api +` + manifestPath := filepath.Join(tempDir, "MANIFEST.MF") + err = os.WriteFile(manifestPath, []byte(manifestContent), 0644) + require.NoError(t, err) + + headers, err := GetManifestHeaders(manifestPath) + require.NoError(t, err) + + assert.Equal(t, "1.0", headers["Manifest-Version"]) + assert.Equal(t, "Test Bundle", headers["Bundle-Name"]) + assert.Equal(t, "com.test.bundle", headers["Bundle-SymbolicName"]) + assert.Equal(t, "1.0.0", headers["Bundle-Version"]) + assert.Equal(t, "javax.xml.bind, javax.xml.stream", headers["Import-Package"]) + assert.Equal(t, "com.test.api", headers["Export-Package"]) +} + +func TestGetManifestHeaders_MultilineContinuation(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + manifestContent := `Manifest-Version: 1.0 +Import-Package: javax.xml.bind, + javax.xml.stream, + javax.xml.transform +Bundle-Name: Test +` + manifestPath := filepath.Join(tempDir, "MANIFEST.MF") + err = os.WriteFile(manifestPath, []byte(manifestContent), 0644) + require.NoError(t, err) + + headers, err := GetManifestHeaders(manifestPath) + require.NoError(t, err) + + // Continuation lines should be merged with spaces + expected := "javax.xml.bind, javax.xml.stream, javax.xml.transform" + assert.Equal(t, expected, headers["Import-Package"]) +} + +func TestGetManifestHeaders_NonExistent(t *testing.T) { + headers, err := GetManifestHeaders("/nonexistent/MANIFEST.MF") + require.NoError(t, err) + assert.Empty(t, headers) +} + +func TestGetManifestHeaders_Empty(t *testing.T) { + tempDir, err := os.MkdirTemp("", "utils-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + manifestPath := filepath.Join(tempDir, "MANIFEST.MF") + err = os.WriteFile(manifestPath, []byte(""), 0644) + require.NoError(t, err) + + headers, err := GetManifestHeaders(manifestPath) + require.NoError(t, err) + assert.Empty(t, headers) +} diff --git a/internal/file/file.go b/internal/file/file.go index 6ff6637..dac2fde 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -278,3 +278,57 @@ func ZipDirToBase64(src string) (string, error) { } return base64.StdEncoding.EncodeToString(fileContent), nil } + +// ReadManifest reads a MANIFEST.MF file and returns key-value pairs +func ReadManifest(manifestPath string) (map[string]string, error) { + metadata := make(map[string]string) + + file, err := os.Open(manifestPath) + if err != nil { + return nil, errors.Wrap(err, 0) + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + return nil, errors.Wrap(err, 0) + } + + lines := strings.Split(string(content), "\n") + var currentKey string + var currentValue strings.Builder + + for _, line := range lines { + line = strings.TrimRight(line, "\r") + + // Multi-line values start with a space + if len(line) > 0 && line[0] == ' ' { + if currentKey != "" { + currentValue.WriteString(strings.TrimPrefix(line, " ")) + } + continue + } + + // Save previous key-value pair if exists + if currentKey != "" { + metadata[currentKey] = currentValue.String() + currentValue.Reset() + } + + // Parse new key-value pair + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + currentKey = strings.TrimSpace(parts[0]) + currentValue.WriteString(strings.TrimSpace(parts[1])) + } else { + currentKey = "" + } + } + + // Save last key-value pair + if currentKey != "" { + metadata[currentKey] = currentValue.String() + } + + return metadata, nil +} diff --git a/internal/httpclnt/batch.go b/internal/httpclnt/batch.go new file mode 100644 index 0000000..e070212 --- /dev/null +++ b/internal/httpclnt/batch.go @@ -0,0 +1,540 @@ +package httpclnt + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "strconv" + "strings" + + "github.com/rs/zerolog/log" +) + +const ( + // DefaultBatchSize is the default number of operations per batch request + DefaultBatchSize = 90 + + // Batch boundary prefixes (must match OData multipart/mixed format) + batchBoundaryPrefix = "batch_" + changesetBoundaryPrefix = "changeset_" +) + +// BatchOperation represents a single operation in a batch request +type BatchOperation struct { + Method string // HTTP method (POST, PUT, DELETE, PATCH, GET) + Path string // API path (e.g., "/api/v1/StringParameters") + Body []byte // Request body (raw bytes - caller handles marshaling) + ContentID string // Content-ID for tracking this operation + Headers map[string]string // Additional headers (e.g., If-Match, Content-Type) + IsQuery bool // True for GET operations (goes in query section, not changeset) +} + +// BatchResponse represents the response from a batch request +type BatchResponse struct { + Operations []BatchOperationResponse +} + +// BatchOperationResponse represents a single operation response +type BatchOperationResponse struct { + ContentID string + StatusCode int + Headers http.Header + Body []byte + Error error +} + +// BatchRequest handles building and executing OData $batch requests +type BatchRequest struct { + exe *HTTPExecuter + operations []BatchOperation + batchBoundary string + changesetBoundary string +} + +// boundaryCounter is used to generate unique boundary strings +var boundaryCounter = 0 + +// NewBatchRequest creates a new batch request builder +func (e *HTTPExecuter) NewBatchRequest() *BatchRequest { + return &BatchRequest{ + exe: e, + operations: make([]BatchOperation, 0), + batchBoundary: generateBoundary(batchBoundaryPrefix), + changesetBoundary: generateBoundary(changesetBoundaryPrefix), + } +} + +// AddOperation adds an operation to the batch +func (br *BatchRequest) AddOperation(op BatchOperation) { + br.operations = append(br.operations, op) +} + +// Execute sends the batch request and returns the responses +func (br *BatchRequest) Execute() (*BatchResponse, error) { + if len(br.operations) == 0 { + return &BatchResponse{Operations: []BatchOperationResponse{}}, nil + } + + // Build multipart batch request body + body, err := br.buildBatchBody() + if err != nil { + return nil, fmt.Errorf("failed to build batch body: %w", err) + } + + // Execute the batch request + contentType := fmt.Sprintf("multipart/mixed; boundary=%s", br.batchBoundary) + headers := map[string]string{ + "Content-Type": contentType, + "Accept": "multipart/mixed", + } + + resp, err := br.exe.ExecRequestWithCookies("POST", "/api/v1/$batch", bytes.NewReader(body), headers, nil) + if err != nil { + return nil, fmt.Errorf("batch request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("batch request failed with status %d: %s", resp.StatusCode, string(bodyBytes)) + } + + // Parse the multipart response + return br.parseBatchResponse(resp) +} + +// ExecuteInBatches splits operations into batches and executes them +func (br *BatchRequest) ExecuteInBatches(batchSize int) (*BatchResponse, error) { + if batchSize <= 0 { + batchSize = DefaultBatchSize + } + + allOps := br.operations + var allResponses []BatchOperationResponse + + for i := 0; i < len(allOps); i += batchSize { + end := i + batchSize + if end > len(allOps) { + end = len(allOps) + } + + // Create a batch for this chunk + batch := br.exe.NewBatchRequest() + batch.operations = allOps[i:end] + + // Execute this batch + resp, err := batch.Execute() + if err != nil { + return nil, fmt.Errorf("batch %d-%d failed: %w", i, end, err) + } + + allResponses = append(allResponses, resp.Operations...) + } + + return &BatchResponse{Operations: allResponses}, nil +} + +// buildBatchBody constructs the multipart batch request body +func (br *BatchRequest) buildBatchBody() ([]byte, error) { + var buf bytes.Buffer + + // Separate query and changeset operations + var queryOps []BatchOperation + var changesetOps []BatchOperation + + for _, op := range br.operations { + if op.IsQuery { + queryOps = append(queryOps, op) + } else { + changesetOps = append(changesetOps, op) + } + } + + // Start batch boundary + fmt.Fprintf(&buf, "--%s\r\n", br.batchBoundary) + + // Add query operations (if any) - these go directly in batch, not in changeset + if len(queryOps) > 0 { + for _, op := range queryOps { + if err := br.writeQueryOperation(&buf, op); err != nil { + return nil, err + } + fmt.Fprintf(&buf, "--%s\r\n", br.batchBoundary) + } + } + + // Add changeset for modifying operations (POST, PUT, DELETE, PATCH) + if len(changesetOps) > 0 { + fmt.Fprintf(&buf, "Content-Type: multipart/mixed; boundary=%s\r\n", br.changesetBoundary) + fmt.Fprintf(&buf, "\r\n") + + // Add each operation as a changeset part + for _, op := range changesetOps { + if err := br.writeChangesetOperation(&buf, op); err != nil { + return nil, err + } + } + + // End changeset boundary + fmt.Fprintf(&buf, "--%s--\r\n", br.changesetBoundary) + fmt.Fprintf(&buf, "\r\n") + } + + // End batch boundary + fmt.Fprintf(&buf, "--%s--\r\n", br.batchBoundary) + + return buf.Bytes(), nil +} + +// writeQueryOperation writes a query (GET) operation to the batch body +func (br *BatchRequest) writeQueryOperation(buf *bytes.Buffer, op BatchOperation) error { + fmt.Fprintf(buf, "Content-Type: application/http\r\n") + fmt.Fprintf(buf, "Content-Transfer-Encoding: binary\r\n") + + if op.ContentID != "" { + fmt.Fprintf(buf, "Content-ID: %s\r\n", op.ContentID) + } + + fmt.Fprintf(buf, "\r\n") + + // HTTP request line + fmt.Fprintf(buf, "%s %s HTTP/1.1\r\n", op.Method, op.Path) + + // Headers + for key, value := range op.Headers { + fmt.Fprintf(buf, "%s: %s\r\n", key, value) + } + + fmt.Fprintf(buf, "\r\n") + + return nil +} + +// writeChangesetOperation writes a changeset operation to the batch body +func (br *BatchRequest) writeChangesetOperation(buf *bytes.Buffer, op BatchOperation) error { + // Changeset part boundary + fmt.Fprintf(buf, "--%s\r\n", br.changesetBoundary) + fmt.Fprintf(buf, "Content-Type: application/http\r\n") + fmt.Fprintf(buf, "Content-Transfer-Encoding: binary\r\n") + + if op.ContentID != "" { + fmt.Fprintf(buf, "Content-ID: %s\r\n", op.ContentID) + } + + fmt.Fprintf(buf, "\r\n") + + // HTTP request line + fmt.Fprintf(buf, "%s %s HTTP/1.1\r\n", op.Method, op.Path) + + // Headers + for key, value := range op.Headers { + fmt.Fprintf(buf, "%s: %s\r\n", key, value) + } + + // Body + if len(op.Body) > 0 { + fmt.Fprintf(buf, "Content-Length: %d\r\n", len(op.Body)) + fmt.Fprintf(buf, "\r\n") + buf.Write(op.Body) + } else { + fmt.Fprintf(buf, "\r\n") + } + + fmt.Fprintf(buf, "\r\n") + + return nil +} + +// parseBatchResponse parses the multipart batch response +func (br *BatchRequest) parseBatchResponse(resp *http.Response) (*BatchResponse, error) { + mediaType, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return nil, fmt.Errorf("failed to parse response content-type: %w", err) + } + + if !strings.HasPrefix(mediaType, "multipart/") { + return nil, fmt.Errorf("expected multipart response, got %s", mediaType) + } + + boundary := params["boundary"] + if boundary == "" { + return nil, fmt.Errorf("no boundary in multipart response") + } + + mr := multipart.NewReader(resp.Body, boundary) + + var operations []BatchOperationResponse + + // Read batch parts + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("failed to read batch part: %w", err) + } + + // Check if this is a changeset + partContentType := part.Header.Get("Content-Type") + if strings.HasPrefix(partContentType, "multipart/mixed") { + // Parse the changeset + changesetOps, err := br.parseChangeset(part) + if err != nil { + return nil, fmt.Errorf("failed to parse changeset: %w", err) + } + operations = append(operations, changesetOps...) + } else if strings.HasPrefix(partContentType, "application/http") { + // Single operation response (query result) + op, err := br.parseOperationResponseFromPart(part) + if err != nil { + op = BatchOperationResponse{Error: err} + } + operations = append(operations, op) + } + } + + return &BatchResponse{Operations: operations}, nil +} + +// parseChangeset parses a changeset multipart section +func (br *BatchRequest) parseChangeset(changesetReader io.Reader) ([]BatchOperationResponse, error) { + // Read the changeset to get its boundary + changesetBytes, err := io.ReadAll(changesetReader) + if err != nil { + return nil, fmt.Errorf("failed to read changeset: %w", err) + } + + // Extract boundary from the first line + lines := strings.Split(string(changesetBytes), "\r\n") + if len(lines) == 0 { + return nil, fmt.Errorf("empty changeset") + } + + // Find the boundary (first line starting with --) + var changesetBoundary string + for _, line := range lines { + if strings.HasPrefix(line, "--") { + changesetBoundary = strings.TrimPrefix(line, "--") + break + } + } + + if changesetBoundary == "" { + return nil, fmt.Errorf("no changeset boundary found") + } + + mr := multipart.NewReader(bytes.NewReader(changesetBytes), changesetBoundary) + + var operations []BatchOperationResponse + + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("failed to read changeset part: %w", err) + } + + op, err := br.parseOperationResponseFromPart(part) + if err != nil { + // Log error but continue with other operations + log.Warn().Msgf("Failed to parse changeset part: %v", err) + op = BatchOperationResponse{Error: err} + } + + operations = append(operations, op) + } + + return operations, nil +} + +// parseOperationResponseFromPart parses a single operation response from a multipart part +func (br *BatchRequest) parseOperationResponseFromPart(part *multipart.Part) (BatchOperationResponse, error) { + contentID := part.Header.Get("Content-Id") + if contentID == "" { + contentID = part.Header.Get("Content-ID") + } + + // Read the HTTP response + bodyBytes, err := io.ReadAll(part) + if err != nil { + return BatchOperationResponse{}, fmt.Errorf("failed to read operation response: %w", err) + } + + // Parse HTTP response + lines := strings.Split(string(bodyBytes), "\r\n") + if len(lines) < 1 { + return BatchOperationResponse{}, fmt.Errorf("invalid HTTP response") + } + + // Parse status line (e.g., "HTTP/1.1 201 Created") + statusLine := lines[0] + parts := strings.SplitN(statusLine, " ", 3) + if len(parts) < 2 { + return BatchOperationResponse{}, fmt.Errorf("invalid status line: %s", statusLine) + } + + statusCode, err := strconv.Atoi(parts[1]) + if err != nil { + return BatchOperationResponse{}, fmt.Errorf("invalid status code: %s", parts[1]) + } + + // Parse headers + headers := make(http.Header) + i := 1 + for ; i < len(lines); i++ { + line := lines[i] + if line == "" { + i++ + break + } + + // Parse header + colonIdx := strings.Index(line, ":") + if colonIdx > 0 { + key := strings.TrimSpace(line[:colonIdx]) + value := strings.TrimSpace(line[colonIdx+1:]) + headers.Add(key, value) + } + } + + // Remaining lines are the body + var body []byte + if i < len(lines) { + bodyStr := strings.Join(lines[i:], "\r\n") + body = []byte(strings.TrimSpace(bodyStr)) + } + + return BatchOperationResponse{ + ContentID: contentID, + StatusCode: statusCode, + Headers: headers, + Body: body, + }, nil +} + +// generateBoundary generates a unique boundary string +func generateBoundary(prefix string) string { + boundaryCounter++ + return fmt.Sprintf("%s%d", prefix, boundaryCounter) +} + +// Helper functions for building batch operations + +// AddCreateStringParameterOp adds a CREATE operation for a string parameter to the batch +func AddCreateStringParameterOp(batch *BatchRequest, pid, id, value, contentID string) { + body := map[string]string{ + "Pid": pid, + "Id": id, + "Value": value, + } + bodyJSON, _ := json.Marshal(body) + + batch.AddOperation(BatchOperation{ + Method: "POST", + Path: "/api/v1/StringParameters", + Body: bodyJSON, + ContentID: contentID, + Headers: map[string]string{ + "Content-Type": "application/json", + }, + }) +} + +// AddUpdateStringParameterOp adds an UPDATE operation for a string parameter to the batch +func AddUpdateStringParameterOp(batch *BatchRequest, pid, id, value, contentID string) { + body := map[string]string{ + "Value": value, + } + bodyJSON, _ := json.Marshal(body) + + path := fmt.Sprintf("/api/v1/StringParameters(Pid='%s',Id='%s')", pid, id) + + batch.AddOperation(BatchOperation{ + Method: "PUT", + Path: path, + Body: bodyJSON, + ContentID: contentID, + Headers: map[string]string{ + "Content-Type": "application/json", + "If-Match": "*", + }, + }) +} + +// AddDeleteStringParameterOp adds a DELETE operation for a string parameter to the batch +func AddDeleteStringParameterOp(batch *BatchRequest, pid, id, contentID string) { + path := fmt.Sprintf("/api/v1/StringParameters(Pid='%s',Id='%s')", pid, id) + + batch.AddOperation(BatchOperation{ + Method: "DELETE", + Path: path, + ContentID: contentID, + Headers: map[string]string{ + "If-Match": "*", + }, + }) +} + +// AddCreateBinaryParameterOp adds a CREATE operation for a binary parameter to the batch +func AddCreateBinaryParameterOp(batch *BatchRequest, pid, id, value, contentType, contentID string) { + body := map[string]string{ + "Pid": pid, + "Id": id, + "Value": value, + "ContentType": contentType, + } + bodyJSON, _ := json.Marshal(body) + + batch.AddOperation(BatchOperation{ + Method: "POST", + Path: "/api/v1/BinaryParameters", + Body: bodyJSON, + ContentID: contentID, + Headers: map[string]string{ + "Content-Type": "application/json", + }, + }) +} + +// AddUpdateBinaryParameterOp adds an UPDATE operation for a binary parameter to the batch +func AddUpdateBinaryParameterOp(batch *BatchRequest, pid, id, value, contentType, contentID string) { + body := map[string]string{ + "Value": value, + "ContentType": contentType, + } + bodyJSON, _ := json.Marshal(body) + + path := fmt.Sprintf("/api/v1/BinaryParameters(Pid='%s',Id='%s')", pid, id) + + batch.AddOperation(BatchOperation{ + Method: "PUT", + Path: path, + Body: bodyJSON, + ContentID: contentID, + Headers: map[string]string{ + "Content-Type": "application/json", + "If-Match": "*", + }, + }) +} + +// AddDeleteBinaryParameterOp adds a DELETE operation for a binary parameter to the batch +func AddDeleteBinaryParameterOp(batch *BatchRequest, pid, id, contentID string) { + path := fmt.Sprintf("/api/v1/BinaryParameters(Pid='%s',Id='%s')", pid, id) + + batch.AddOperation(BatchOperation{ + Method: "DELETE", + Path: path, + ContentID: contentID, + Headers: map[string]string{ + "If-Match": "*", + }, + }) +} diff --git a/internal/models/configure.go b/internal/models/configure.go new file mode 100644 index 0000000..693cbb6 --- /dev/null +++ b/internal/models/configure.go @@ -0,0 +1,85 @@ +package models + +// ConfigureConfig represents the complete configuration file structure +type ConfigureConfig struct { + DeploymentPrefix string `yaml:"deploymentPrefix,omitempty"` + Packages []ConfigurePackage `yaml:"packages"` +} + +// ConfigurePackage represents a package containing artifacts to configure +type ConfigurePackage struct { + ID string `yaml:"integrationSuiteId"` + DisplayName string `yaml:"displayName,omitempty"` + Deploy bool `yaml:"deploy"` // Deploy all artifacts in package after configuration + Artifacts []ConfigureArtifact `yaml:"artifacts"` +} + +func (p *ConfigurePackage) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Set defaults + type rawPackage ConfigurePackage + raw := rawPackage{ + Deploy: false, // By default, don't deploy unless explicitly requested + } + + if err := unmarshal(&raw); err != nil { + return err + } + + *p = ConfigurePackage(raw) + return nil +} + +// ConfigureArtifact represents an artifact with its configuration parameters +type ConfigureArtifact struct { + ID string `yaml:"artifactId"` + DisplayName string `yaml:"displayName,omitempty"` + Type string `yaml:"type"` // Integration, MessageMapping, ScriptCollection, ValueMapping + Version string `yaml:"version,omitempty"` // Artifact version, defaults to "active" + Deploy bool `yaml:"deploy"` // Deploy this specific artifact after configuration + Parameters []ConfigurationParameter `yaml:"parameters,omitempty"` // List of configuration parameters to update + Batch *BatchSettings `yaml:"batch,omitempty"` // Optional batch processing settings +} + +func (a *ConfigureArtifact) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Set defaults + type rawArtifact ConfigureArtifact + raw := rawArtifact{ + Version: "active", + Deploy: false, // By default, don't deploy unless explicitly requested + } + + if err := unmarshal(&raw); err != nil { + return err + } + + *a = ConfigureArtifact(raw) + return nil +} + +// ConfigurationParameter represents a single configuration parameter to update +type ConfigurationParameter struct { + Key string `yaml:"key"` + Value string `yaml:"value"` +} + +// BatchSettings allows per-artifact batch configuration +type BatchSettings struct { + Enabled bool `yaml:"enabled"` // Enable batch processing for this artifact + BatchSize int `yaml:"batchSize,omitempty"` // Number of parameters per batch request +} + +func (b *BatchSettings) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Set defaults + type rawBatch BatchSettings + raw := rawBatch{ + Enabled: true, + BatchSize: 90, // Default batch size from batch.go + } + + if err := unmarshal(&raw); err != nil { + return err + } + + *b = BatchSettings(raw) + return nil +} diff --git a/internal/models/deploy.go b/internal/models/deploy.go new file mode 100644 index 0000000..de2c9f1 --- /dev/null +++ b/internal/models/deploy.go @@ -0,0 +1,93 @@ +package models + +// OrchestratorConfig represents orchestrator-specific settings +type OrchestratorConfig struct { + PackagesDir string `yaml:"packagesDir"` + DeployConfig string `yaml:"deployConfig"` + DeploymentPrefix string `yaml:"deploymentPrefix,omitempty"` + PackageFilter string `yaml:"packageFilter,omitempty"` + ArtifactFilter string `yaml:"artifactFilter,omitempty"` + ConfigPattern string `yaml:"configPattern,omitempty"` + MergeConfigs bool `yaml:"mergeConfigs,omitempty"` + KeepTemp bool `yaml:"keepTemp,omitempty"` + Mode string `yaml:"mode,omitempty"` // "update-and-deploy", "update-only", "deploy-only" + // Deployment settings + DeployRetries int `yaml:"deployRetries,omitempty"` + DeployDelaySeconds int `yaml:"deployDelaySeconds,omitempty"` + ParallelDeployments int `yaml:"parallelDeployments,omitempty"` +} + +// DeployConfig represents the complete deployment configuration +type DeployConfig struct { + DeploymentPrefix string `yaml:"deploymentPrefix"` + Packages []Package `yaml:"packages"` + Orchestrator *OrchestratorConfig `yaml:"orchestrator,omitempty"` +} + +// Package represents a SAP CPI package +type Package struct { + ID string `yaml:"integrationSuiteId"` + PackageDir string `yaml:"packageDir,omitempty"` + DisplayName string `yaml:"displayName,omitempty"` + Description string `yaml:"description,omitempty"` + ShortText string `yaml:"short_text,omitempty"` + Sync bool `yaml:"sync"` + Deploy bool `yaml:"deploy"` + Artifacts []Artifact `yaml:"artifacts"` +} + +func (p *Package) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Set defaults + type rawPackage Package + raw := rawPackage{ + Sync: true, + Deploy: true, + } + + if err := unmarshal(&raw); err != nil { + return err + } + + *p = Package(raw) + return nil +} + +// Artifact represents a SAP CPI artifact (Integration Flow, Script Collection, etc.) +type Artifact struct { + Id string `yaml:"artifactId"` + ArtifactDir string `yaml:"artifactDir"` + DisplayName string `yaml:"displayName"` + Type string `yaml:"type"` + Sync bool `yaml:"sync"` + Deploy bool `yaml:"deploy"` + ConfigOverrides map[string]interface{} `yaml:"configOverrides"` +} + +func (a *Artifact) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Set defaults + type rawArtifact Artifact + raw := rawArtifact{ + Sync: true, + Deploy: true, + } + + if err := unmarshal(&raw); err != nil { + return err + } + + *a = Artifact(raw) + return nil +} + +// PackageMetadata represents metadata extracted from {PackageName}.json +type PackageMetadata struct { + ID string `json:"Id"` + Name string `json:"Name"` + Description string `json:"Description"` + ShortText string `json:"ShortText"` +} + +// PackageJSON represents the structure of {PackageName}.json files +type PackageJSON struct { + D PackageMetadata `json:"d"` +} diff --git a/internal/repo/partnerdirectory.go b/internal/repo/partnerdirectory.go new file mode 100644 index 0000000..2e1273f --- /dev/null +++ b/internal/repo/partnerdirectory.go @@ -0,0 +1,477 @@ +package repo + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/engswee/flashpipe/internal/api" + "github.com/rs/zerolog/log" +) + +const ( + stringPropertiesFile = "String.properties" + binaryDirName = "Binary" + metadataFileName = "_metadata.json" + defaultBinaryExt = "bin" +) + +// supportedContentTypes defines the valid content types that SAP CPI uses +// These are simple type strings (not MIME types) +var supportedContentTypes = map[string]bool{ + "xml": true, + "xsl": true, + "xsd": true, + "json": true, + "txt": true, + "zip": true, + "gz": true, + "zlib": true, + "crt": true, +} + +// PartnerDirectory handles Partner Directory file operations +type PartnerDirectory struct { + ResourcesPath string +} + +// NewPartnerDirectory creates a new Partner Directory repository +func NewPartnerDirectory(resourcesPath string) *PartnerDirectory { + return &PartnerDirectory{ + ResourcesPath: resourcesPath, + } +} + +// GetLocalPIDs returns all PIDs that have local directories +func (pd *PartnerDirectory) GetLocalPIDs() ([]string, error) { + entries, err := os.ReadDir(pd.ResourcesPath) + if err != nil { + if os.IsNotExist(err) { + return []string{}, nil + } + return nil, fmt.Errorf("failed to read resources directory: %w", err) + } + + var pids []string + for _, entry := range entries { + if entry.IsDir() { + pids = append(pids, entry.Name()) + } + } + + sort.Strings(pids) + return pids, nil +} + +// WriteStringParameters writes string parameters to a properties file +func (pd *PartnerDirectory) WriteStringParameters(pid string, params []api.StringParameter, replace bool) error { + pidDir := filepath.Join(pd.ResourcesPath, pid) + if err := os.MkdirAll(pidDir, 0755); err != nil { + return fmt.Errorf("failed to create PID directory: %w", err) + } + + propertiesFile := filepath.Join(pidDir, stringPropertiesFile) + + if replace || !fileExists(propertiesFile) { + if err := writePropertiesFile(propertiesFile, params); err != nil { + return err + } + log.Debug().Msgf("Created/Updated %s for PID %s", stringPropertiesFile, pid) + } else { + addedCount, err := mergePropertiesFile(propertiesFile, params) + if err != nil { + return err + } + log.Debug().Msgf("Merged %d new values into %s for PID %s", addedCount, stringPropertiesFile, pid) + } + + return nil +} + +// WriteBinaryParameters writes binary parameters to files +func (pd *PartnerDirectory) WriteBinaryParameters(pid string, params []api.BinaryParameter, replace bool) error { + pidDir := filepath.Join(pd.ResourcesPath, pid) + binaryDir := filepath.Join(pidDir, binaryDirName) + + if err := os.MkdirAll(binaryDir, 0755); err != nil { + return fmt.Errorf("failed to create binary directory: %w", err) + } + + for _, param := range params { + filePath := filepath.Join(binaryDir, param.ID) + + // Check if file exists + exists := fileExists(filePath) + + // Skip if not replacing and file exists + if !replace && exists { + log.Debug().Msgf("Skipping existing binary parameter %s/%s", pid, param.ID) + continue + } + + if err := saveBinaryParameterToFile(binaryDir, param); err != nil { + return fmt.Errorf("failed to save binary parameter %s: %w", param.ID, err) + } + + if err := updateMetadataFile(binaryDir, param.ID, param.ContentType); err != nil { + return fmt.Errorf("failed to update metadata: %w", err) + } + } + + return nil +} + +// ReadStringParameters reads string parameters from a properties file +func (pd *PartnerDirectory) ReadStringParameters(pid string) ([]api.StringParameter, error) { + propertiesFile := filepath.Join(pd.ResourcesPath, pid, stringPropertiesFile) + + if !fileExists(propertiesFile) { + return []api.StringParameter{}, nil + } + + return readPropertiesFile(propertiesFile, pid) +} + +// ReadBinaryParameters reads binary parameters from files +func (pd *PartnerDirectory) ReadBinaryParameters(pid string) ([]api.BinaryParameter, error) { + binaryDir := filepath.Join(pd.ResourcesPath, pid, binaryDirName) + + if !dirExists(binaryDir) { + return []api.BinaryParameter{}, nil + } + + // Read metadata + metadataPath := filepath.Join(binaryDir, metadataFileName) + metadata := make(map[string]string) + if fileExists(metadataPath) { + data, err := os.ReadFile(metadataPath) + if err != nil { + return nil, fmt.Errorf("failed to read metadata file: %w", err) + } + if err := json.Unmarshal(data, &metadata); err != nil { + return nil, fmt.Errorf("failed to parse metadata: %w", err) + } + } + + // Read all binary files + entries, err := os.ReadDir(binaryDir) + if err != nil { + return nil, fmt.Errorf("failed to read binary directory: %w", err) + } + + var params []api.BinaryParameter + seenParams := make(map[string]bool) + + for _, entry := range entries { + if entry.IsDir() || entry.Name() == metadataFileName { + continue + } + + filePath := filepath.Join(binaryDir, entry.Name()) + + // Use filename without extension as ID + paramID := removeFileExtension(entry.Name()) + + // Check for duplicates (same ID, different extension) + if seenParams[paramID] { + log.Warn().Msgf("Duplicate binary parameter %s/%s - skipping file %s", pid, paramID, entry.Name()) + continue + } + seenParams[paramID] = true + + data, err := os.ReadFile(filePath) + if err != nil { + log.Warn().Msgf("Failed to read binary file %s: %v", entry.Name(), err) + continue + } + + // Encode to base64 + encoded := base64.StdEncoding.EncodeToString(data) + + // Get full content type from metadata (includes encoding if present) + contentType := metadata[entry.Name()] + if contentType == "" { + // Infer from extension if not in metadata + ext := strings.TrimPrefix(filepath.Ext(entry.Name()), ".") + if ext == "" { + ext = defaultBinaryExt + } + contentType = ext + } + + log.Debug().Msgf("Loaded binary parameter %s/%s (%s, %d bytes)", pid, paramID, contentType, len(data)) + + params = append(params, api.BinaryParameter{ + Pid: pid, + ID: paramID, + Value: encoded, + ContentType: contentType, + }) + } + + return params, nil +} + +// Helper functions + +func writePropertiesFile(filePath string, params []api.StringParameter) error { + // Sort by ID for consistent output + sort.Slice(params, func(i, j int) bool { + return params[i].ID < params[j].ID + }) + + var content strings.Builder + for _, param := range params { + content.WriteString(fmt.Sprintf("%s=%s\n", param.ID, escapePropertyValue(param.Value))) + } + + if err := os.WriteFile(filePath, []byte(content.String()), 0644); err != nil { + return fmt.Errorf("failed to write properties file: %w", err) + } + + return nil +} + +func mergePropertiesFile(filePath string, newParams []api.StringParameter) (int, error) { + // Read existing properties + existing := make(map[string]string) + if fileExists(filePath) { + data, err := os.ReadFile(filePath) + if err != nil { + return 0, fmt.Errorf("failed to read existing properties: %w", err) + } + + lines := strings.Split(string(data), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + existing[parts[0]] = parts[1] + } + } + } + + // Add new parameters + addedCount := 0 + for _, param := range newParams { + if _, exists := existing[param.ID]; !exists { + existing[param.ID] = escapePropertyValue(param.Value) + addedCount++ + } + } + + // Write back sorted + keys := make([]string, 0, len(existing)) + for k := range existing { + keys = append(keys, k) + } + sort.Strings(keys) + + var content strings.Builder + for _, key := range keys { + content.WriteString(fmt.Sprintf("%s=%s\n", key, existing[key])) + } + + if err := os.WriteFile(filePath, []byte(content.String()), 0644); err != nil { + return 0, fmt.Errorf("failed to write properties file: %w", err) + } + + return addedCount, nil +} + +func readPropertiesFile(filePath string, pid string) ([]api.StringParameter, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read properties file: %w", err) + } + + var params []api.StringParameter + lines := strings.Split(string(data), "\n") + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + params = append(params, api.StringParameter{ + Pid: pid, + ID: parts[0], + Value: unescapePropertyValue(parts[1]), + }) + } + } + + return params, nil +} + +func saveBinaryParameterToFile(binaryDir string, param api.BinaryParameter) error { + // Decode base64 + data, err := base64.StdEncoding.DecodeString(param.Value) + if err != nil { + return fmt.Errorf("failed to decode base64: %w", err) + } + + // Determine file extension from content type + log.Debug().Msgf("Processing binary parameter %s with contentType: %s", param.ID, param.ContentType) + ext := getFileExtension(param.ContentType) + log.Debug().Msgf("Determined file extension: %s", ext) + + // Create filename: {ParamId}.{ext} + filename := param.ID + if ext != "" && !strings.HasSuffix(strings.ToLower(filename), "."+ext) { + filename = fmt.Sprintf("%s.%s", param.ID, ext) + } + + filePath := filepath.Join(binaryDir, filename) + + if err := os.WriteFile(filePath, data, 0644); err != nil { + return fmt.Errorf("failed to write binary file: %w", err) + } + + log.Info().Msgf("Saved binary parameter: %s (%s, %d bytes)", filename, param.ContentType, len(data)) + return nil +} + +func updateMetadataFile(binaryDir string, paramID string, contentType string) error { + // Only store in metadata if contentType has encoding/parameters (contains semicolon) + if !strings.Contains(contentType, ";") { + return nil + } + + metadataPath := filepath.Join(binaryDir, metadataFileName) + + metadata := make(map[string]string) + if fileExists(metadataPath) { + data, err := os.ReadFile(metadataPath) + if err != nil { + return fmt.Errorf("failed to read metadata: %w", err) + } + if err := json.Unmarshal(data, &metadata); err != nil { + return fmt.Errorf("failed to parse metadata: %w", err) + } + } + + // Determine filename + ext := getFileExtension(contentType) + filename := paramID + if ext != "" && !strings.HasSuffix(strings.ToLower(filename), "."+ext) { + filename = fmt.Sprintf("%s.%s", paramID, ext) + } + + // Store full content type (with encoding) + metadata[filename] = contentType + + data, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal metadata: %w", err) + } + + if err := os.WriteFile(metadataPath, data, 0644); err != nil { + return fmt.Errorf("failed to write metadata: %w", err) + } + + return nil +} + +func parseContentType(contentType string) (string, string) { + // SAP CPI returns simple types like "xml", "json", "txt" + // But may also include encoding like "xml; encoding=UTF-8" + + // Remove encoding/parameters (e.g., "xml; encoding=UTF-8" -> "xml") + baseType := contentType + if idx := strings.Index(contentType, ";"); idx > 0 { + baseType = strings.TrimSpace(contentType[:idx]) + } + + // If it's a MIME type like "text/xml" or "application/json", extract the subtype + if strings.Contains(baseType, "/") { + parts := strings.Split(baseType, "/") + if len(parts) == 2 { + ext := parts[1] + // Handle special cases like "application/octet-stream" + if ext == "octet-stream" { + return defaultBinaryExt, contentType + } + return ext, contentType + } + } + + // Otherwise it's already a simple type like "xml", "json", etc. + return baseType, contentType +} + +func escapePropertyValue(value string) string { + value = strings.ReplaceAll(value, "\\", "\\\\") + value = strings.ReplaceAll(value, "\n", "\\n") + value = strings.ReplaceAll(value, "\r", "\\r") + return value +} + +func unescapePropertyValue(value string) string { + value = strings.ReplaceAll(value, "\\n", "\n") + value = strings.ReplaceAll(value, "\\r", "\r") + value = strings.ReplaceAll(value, "\\\\", "\\") + return value +} + +func getFileExtension(contentType string) string { + ext, _ := parseContentType(contentType) + // Use the extension if it's in our supported list or if it's reasonable + if isValidContentType(ext) { + return ext + } + // If not in supported list but looks valid (alphanumeric, 2-5 chars), still use it + if ext != "" && len(ext) >= 2 && len(ext) <= 5 && isAlphanumeric(ext) { + log.Debug().Msgf("Using non-standard extension: %s", ext) + return ext + } + return defaultBinaryExt +} + +func isAlphanumeric(s string) bool { + for _, c := range s { + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + return false + } + } + return true +} + +func removeFileExtension(filename string) string { + ext := filepath.Ext(filename) + if ext != "" { + return strings.TrimSuffix(filename, ext) + } + return filename +} + +func isValidContentType(ext string) bool { + return supportedContentTypes[strings.ToLower(ext)] +} + +func fileExists(path string) bool { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return false + } + return err == nil && !info.IsDir() +} + +func dirExists(path string) bool { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return false + } + return err == nil && info.IsDir() +} diff --git a/internal/repo/partnerdirectory_test.go b/internal/repo/partnerdirectory_test.go new file mode 100644 index 0000000..f7307f1 --- /dev/null +++ b/internal/repo/partnerdirectory_test.go @@ -0,0 +1,708 @@ +package repo + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/engswee/flashpipe/internal/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseContentType_SimpleTypes(t *testing.T) { + tests := []struct { + name string + contentType string + wantExt string + wantFull string + }{ + { + name: "simple xml", + contentType: "xml", + wantExt: "xml", + wantFull: "xml", + }, + { + name: "simple json", + contentType: "json", + wantExt: "json", + wantFull: "json", + }, + { + name: "simple txt", + contentType: "txt", + wantExt: "txt", + wantFull: "txt", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ext, full := parseContentType(tt.contentType) + assert.Equal(t, tt.wantExt, ext) + assert.Equal(t, tt.wantFull, full) + }) + } +} + +func TestParseContentType_WithEncoding(t *testing.T) { + tests := []struct { + name string + contentType string + wantExt string + wantFull string + }{ + { + name: "xml with encoding", + contentType: "xml; encoding=UTF-8", + wantExt: "xml", + wantFull: "xml; encoding=UTF-8", + }, + { + name: "json with charset", + contentType: "json; charset=utf-8", + wantExt: "json", + wantFull: "json; charset=utf-8", + }, + { + name: "xml with multiple parameters", + contentType: "xml; encoding=UTF-8; version=1.0", + wantExt: "xml", + wantFull: "xml; encoding=UTF-8; version=1.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ext, full := parseContentType(tt.contentType) + assert.Equal(t, tt.wantExt, ext) + assert.Equal(t, tt.wantFull, full) + }) + } +} + +func TestParseContentType_MIMETypes(t *testing.T) { + tests := []struct { + name string + contentType string + wantExt string + }{ + { + name: "text/xml", + contentType: "text/xml", + wantExt: "xml", + }, + { + name: "application/json", + contentType: "application/json", + wantExt: "json", + }, + { + name: "application/xml", + contentType: "application/xml", + wantExt: "xml", + }, + { + name: "text/plain", + contentType: "text/plain", + wantExt: "plain", + }, + { + name: "application/octet-stream", + contentType: "application/octet-stream", + wantExt: defaultBinaryExt, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ext, _ := parseContentType(tt.contentType) + assert.Equal(t, tt.wantExt, ext) + }) + } +} + +func TestGetFileExtension_SupportedTypes(t *testing.T) { + tests := []struct { + name string + contentType string + wantExt string + }{ + {"xml", "xml", "xml"}, + {"json", "json", "json"}, + {"xsl", "xsl", "xsl"}, + {"xsd", "xsd", "xsd"}, + {"txt", "txt", "txt"}, + {"zip", "zip", "zip"}, + {"crt", "crt", "crt"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ext := getFileExtension(tt.contentType) + assert.Equal(t, tt.wantExt, ext) + }) + } +} + +func TestGetFileExtension_UnsupportedTypes(t *testing.T) { + tests := []struct { + name string + contentType string + wantExt string + }{ + { + name: "unknown simple type", + contentType: "unknown", + wantExt: defaultBinaryExt, + }, + { + name: "empty", + contentType: "", + wantExt: defaultBinaryExt, + }, + { + name: "too long", + contentType: "verylongextension", + wantExt: defaultBinaryExt, + }, + { + name: "special characters", + contentType: "xml$%", + wantExt: defaultBinaryExt, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ext := getFileExtension(tt.contentType) + assert.Equal(t, tt.wantExt, ext) + }) + } +} + +func TestGetFileExtension_CustomValidTypes(t *testing.T) { + // Non-standard but valid alphanumeric extensions (2-5 chars) + tests := []struct { + name string + contentType string + wantExt string + }{ + {"pdf", "pdf", "pdf"}, + {"docx", "docx", "docx"}, + {"html", "html", "html"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ext := getFileExtension(tt.contentType) + assert.Equal(t, tt.wantExt, ext) + }) + } +} + +func TestEscapeUnescapePropertyValue(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "simple value", + input: "simple", + expected: "simple", + }, + { + name: "with newline", + input: "line1\nline2", + expected: "line1\\nline2", + }, + { + name: "with carriage return", + input: "line1\rline2", + expected: "line1\\rline2", + }, + { + name: "with backslash", + input: "path\\to\\file", + expected: "path\\\\to\\\\file", + }, + { + name: "with all special chars", + input: "line1\nline2\rline3\\backslash", + expected: "line1\\nline2\\rline3\\\\backslash", + }, + } + + for _, tt := range tests { + t.Run(tt.name+" escape", func(t *testing.T) { + escaped := escapePropertyValue(tt.input) + assert.Equal(t, tt.expected, escaped) + }) + + t.Run(tt.name+" unescape", func(t *testing.T) { + unescaped := unescapePropertyValue(tt.expected) + assert.Equal(t, tt.input, unescaped) + }) + + t.Run(tt.name+" roundtrip", func(t *testing.T) { + roundtrip := unescapePropertyValue(escapePropertyValue(tt.input)) + assert.Equal(t, tt.input, roundtrip) + }) + } +} + +func TestRemoveFileExtension(t *testing.T) { + tests := []struct { + name string + filename string + want string + }{ + {"with extension", "file.xml", "file"}, + {"with multiple dots", "file.backup.xml", "file.backup"}, + {"no extension", "file", "file"}, + {"hidden file", ".gitignore", ""}, + {"multiple extensions", "archive.tar.gz", "archive.tar"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := removeFileExtension(tt.filename) + assert.Equal(t, tt.want, result) + }) + } +} + +func TestIsAlphanumeric(t *testing.T) { + tests := []struct { + name string + input string + want bool + }{ + {"letters only", "xml", true}, + {"mixed case", "XmL", true}, + {"with numbers", "file123", true}, + {"with dash", "file-name", false}, + {"with underscore", "file_name", false}, + {"with dot", "file.ext", false}, + {"with space", "file name", false}, + {"empty", "", true}, + {"special chars", "file$", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isAlphanumeric(tt.input) + assert.Equal(t, tt.want, result) + }) + } +} + +func TestWriteAndReadStringParameters(t *testing.T) { + // Create temp directory + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + pid := "TestPID" + + params := []api.StringParameter{ + {Pid: pid, ID: "param1", Value: "value1"}, + {Pid: pid, ID: "param2", Value: "value with\nnewline"}, + {Pid: pid, ID: "param3", Value: "value\\with\\backslash"}, + } + + // Write parameters + err = pd.WriteStringParameters(pid, params, true) + require.NoError(t, err) + + // Read parameters back + readParams, err := pd.ReadStringParameters(pid) + require.NoError(t, err) + + // Verify + assert.Equal(t, len(params), len(readParams)) + for i, param := range params { + assert.Equal(t, param.ID, readParams[i].ID) + assert.Equal(t, param.Value, readParams[i].Value) + assert.Equal(t, pid, readParams[i].Pid) + } +} + +func TestWriteStringParameters_MergeMode(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + pid := "TestPID" + + // Write initial parameters + initial := []api.StringParameter{ + {Pid: pid, ID: "param1", Value: "value1"}, + {Pid: pid, ID: "param2", Value: "value2"}, + } + err = pd.WriteStringParameters(pid, initial, true) + require.NoError(t, err) + + // Merge new parameters (replace=false) + additional := []api.StringParameter{ + {Pid: pid, ID: "param3", Value: "value3"}, + {Pid: pid, ID: "param1", Value: "updated_value1"}, // Should be ignored + } + err = pd.WriteStringParameters(pid, additional, false) + require.NoError(t, err) + + // Read back + readParams, err := pd.ReadStringParameters(pid) + require.NoError(t, err) + + // Verify merge behavior + assert.Equal(t, 3, len(readParams)) + + paramMap := make(map[string]string) + for _, p := range readParams { + paramMap[p.ID] = p.Value + } + + assert.Equal(t, "value1", paramMap["param1"]) // Original should be preserved + assert.Equal(t, "value2", paramMap["param2"]) + assert.Equal(t, "value3", paramMap["param3"]) // New param should be added +} + +func TestWriteAndReadBinaryParameters(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + pid := "TestPID" + + testData := []byte("test") + encoded := base64.StdEncoding.EncodeToString(testData) + + params := []api.BinaryParameter{ + {Pid: pid, ID: "config", Value: encoded, ContentType: "xml"}, + {Pid: pid, ID: "schema", Value: encoded, ContentType: "xsd"}, + } + + // Write parameters + err = pd.WriteBinaryParameters(pid, params, true) + require.NoError(t, err) + + // Verify files exist + configFile := filepath.Join(tempDir, pid, "Binary", "config.xml") + schemaFile := filepath.Join(tempDir, pid, "Binary", "schema.xsd") + assert.True(t, fileExists(configFile)) + assert.True(t, fileExists(schemaFile)) + + // Read parameters back + readParams, err := pd.ReadBinaryParameters(pid) + require.NoError(t, err) + + // Verify + assert.Equal(t, 2, len(readParams)) + + paramMap := make(map[string]api.BinaryParameter) + for _, p := range readParams { + paramMap[p.ID] = p + } + + assert.Equal(t, "xml", paramMap["config"].ContentType) + assert.Equal(t, "xsd", paramMap["schema"].ContentType) + assert.Equal(t, encoded, paramMap["config"].Value) + assert.Equal(t, encoded, paramMap["schema"].Value) +} + +func TestBinaryParameterWithEncoding(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + pid := "TestPID" + + testData := []byte("test") + encoded := base64.StdEncoding.EncodeToString(testData) + + params := []api.BinaryParameter{ + {Pid: pid, ID: "config", Value: encoded, ContentType: "xml; encoding=UTF-8"}, + } + + // Write parameter + err = pd.WriteBinaryParameters(pid, params, true) + require.NoError(t, err) + + // Verify metadata file was created + metadataFile := filepath.Join(tempDir, pid, "Binary", metadataFileName) + assert.True(t, fileExists(metadataFile)) + + // Read metadata + metadataBytes, err := os.ReadFile(metadataFile) + require.NoError(t, err) + + var metadata map[string]string + err = json.Unmarshal(metadataBytes, &metadata) + require.NoError(t, err) + + // Verify metadata contains full content type + assert.Equal(t, "xml; encoding=UTF-8", metadata["config.xml"]) + + // Read parameter back + readParams, err := pd.ReadBinaryParameters(pid) + require.NoError(t, err) + + assert.Equal(t, 1, len(readParams)) + assert.Equal(t, "xml; encoding=UTF-8", readParams[0].ContentType) + assert.Equal(t, encoded, readParams[0].Value) +} + +func TestBinaryParameterWithoutEncoding_NoMetadata(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + pid := "TestPID" + + testData := []byte("{\"key\": \"value\"}") + encoded := base64.StdEncoding.EncodeToString(testData) + + params := []api.BinaryParameter{ + {Pid: pid, ID: "config", Value: encoded, ContentType: "json"}, + } + + // Write parameter + err = pd.WriteBinaryParameters(pid, params, true) + require.NoError(t, err) + + // Verify metadata file was NOT created (since no encoding) + metadataFile := filepath.Join(tempDir, pid, "Binary", metadataFileName) + assert.False(t, fileExists(metadataFile)) +} + +func TestGetLocalPIDs(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + + // Create some PID directories + pids := []string{"PID001", "PID002", "ZZTEST"} + for _, pid := range pids { + err := os.MkdirAll(filepath.Join(tempDir, pid), 0755) + require.NoError(t, err) + } + + // Create a file (should be ignored) + err = os.WriteFile(filepath.Join(tempDir, "notapid.txt"), []byte("test"), 0644) + require.NoError(t, err) + + // Get local PIDs + localPIDs, err := pd.GetLocalPIDs() + require.NoError(t, err) + + // Verify PIDs are returned sorted + assert.Equal(t, []string{"PID001", "PID002", "ZZTEST"}, localPIDs) +} + +func TestGetLocalPIDs_EmptyDirectory(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + + localPIDs, err := pd.GetLocalPIDs() + require.NoError(t, err) + assert.Empty(t, localPIDs) +} + +func TestGetLocalPIDs_NonExistentDirectory(t *testing.T) { + pd := NewPartnerDirectory("/nonexistent/path") + + localPIDs, err := pd.GetLocalPIDs() + require.NoError(t, err) + assert.Empty(t, localPIDs) +} + +func TestReadStringParameters_NonExistent(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + + params, err := pd.ReadStringParameters("NonExistentPID") + require.NoError(t, err) + assert.Empty(t, params) +} + +func TestReadBinaryParameters_NonExistent(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + + params, err := pd.ReadBinaryParameters("NonExistentPID") + require.NoError(t, err) + assert.Empty(t, params) +} + +func TestBinaryParameters_DuplicateHandling(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pid := "TestPID" + binaryDir := filepath.Join(tempDir, pid, "Binary") + err = os.MkdirAll(binaryDir, 0755) + require.NoError(t, err) + + // Create duplicate files with different extensions but same base name + testData := []byte("test data") + err = os.WriteFile(filepath.Join(binaryDir, "config.xml"), testData, 0644) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(binaryDir, "config.txt"), testData, 0644) + require.NoError(t, err) + + pd := NewPartnerDirectory(tempDir) + + // Read should handle duplicates (only return one) + params, err := pd.ReadBinaryParameters(pid) + require.NoError(t, err) + + // Should only get one parameter (the first one encountered) + assert.Equal(t, 1, len(params)) + assert.Equal(t, "config", params[0].ID) +} + +func TestWriteStringParameters_Sorted(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + pd := NewPartnerDirectory(tempDir) + pid := "TestPID" + + // Write parameters in random order + params := []api.StringParameter{ + {Pid: pid, ID: "zzz", Value: "last"}, + {Pid: pid, ID: "aaa", Value: "first"}, + {Pid: pid, ID: "mmm", Value: "middle"}, + } + + err = pd.WriteStringParameters(pid, params, true) + require.NoError(t, err) + + // Read file content + propertiesFile := filepath.Join(tempDir, pid, stringPropertiesFile) + content, err := os.ReadFile(propertiesFile) + require.NoError(t, err) + + // Verify alphabetical order + lines := string(content) + assert.Contains(t, lines, "aaa=first") + assert.Contains(t, lines, "mmm=middle") + assert.Contains(t, lines, "zzz=last") + + // First occurrence should be 'aaa' + assert.True(t, func() bool { + aaaIndex := -1 + mmmIndex := -1 + zzzIndex := -1 + for i, line := range []string{"aaa=first", "mmm=middle", "zzz=last"} { + idx := indexOf(lines, line) + if i == 0 { + aaaIndex = idx + } else if i == 1 { + mmmIndex = idx + } else { + zzzIndex = idx + } + } + return aaaIndex < mmmIndex && mmmIndex < zzzIndex + }()) +} + +func indexOf(s, substr string) int { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return i + } + } + return -1 +} + +func TestFileExists(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create a file + testFile := filepath.Join(tempDir, "test.txt") + err = os.WriteFile(testFile, []byte("test"), 0644) + require.NoError(t, err) + + // Create a directory + testDir := filepath.Join(tempDir, "testdir") + err = os.MkdirAll(testDir, 0755) + require.NoError(t, err) + + assert.True(t, fileExists(testFile)) + assert.False(t, fileExists(testDir)) // Directory should return false + assert.False(t, fileExists(filepath.Join(tempDir, "nonexistent.txt"))) +} + +func TestDirExists(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pd-test-*") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create a file + testFile := filepath.Join(tempDir, "test.txt") + err = os.WriteFile(testFile, []byte("test"), 0644) + require.NoError(t, err) + + // Create a directory + testDir := filepath.Join(tempDir, "testdir") + err = os.MkdirAll(testDir, 0755) + require.NoError(t, err) + + assert.True(t, dirExists(testDir)) + assert.False(t, dirExists(testFile)) // File should return false + assert.False(t, dirExists(filepath.Join(tempDir, "nonexistent"))) +} + +func TestIsValidContentType(t *testing.T) { + tests := []struct { + ext string + valid bool + }{ + {"xml", true}, + {"json", true}, + {"xsl", true}, + {"xsd", true}, + {"txt", true}, + {"zip", true}, + {"gz", true}, + {"zlib", true}, + {"crt", true}, + {"unknown", false}, + {"pdf", false}, + {"", false}, + } + + for _, tt := range tests { + t.Run(tt.ext, func(t *testing.T) { + result := isValidContentType(tt.ext) + assert.Equal(t, tt.valid, result) + }) + } +} diff --git a/internal/sync/synchroniser.go b/internal/sync/synchroniser.go index fb0692e..245aa23 100644 --- a/internal/sync/synchroniser.go +++ b/internal/sync/synchroniser.go @@ -45,15 +45,18 @@ func (s *Synchroniser) PackageToGit(packageDataFromTenant *api.PackageSingleData if err != nil { return errors.Wrap(err, 0) } - defer f.Close() content, err := json.MarshalIndent(packageDataFromTenant, "", " ") if err != nil { + f.Close() return errors.Wrap(err, 0) } _, err = f.Write(content) if err != nil { + f.Close() return errors.Wrap(err, 0) } + // Explicitly close the file before CopyFile to prevent Windows file locking issues + f.Close() // Get existing package details file if it exists and compare values gitSourceFile := fmt.Sprintf("%v/%v.json", artifactsDir, packageId)