diff --git a/.github/workflows/advanced-ci-matrix.yml b/.github/workflows/advanced-ci-matrix.yml new file mode 100644 index 00000000000..cab895f8c63 --- /dev/null +++ b/.github/workflows/advanced-ci-matrix.yml @@ -0,0 +1,391 @@ +--- +name: Advanced CI Matrix Build + +on: + push: + branches: [main, 'copilot/**', 'feature/**', 'dev/**'] + pull_request: + branches: [main] + schedule: + # Run nightly builds at 2 AM UTC + - cron: '0 2 * * *' + workflow_dispatch: + inputs: + build_targets: + description: 'Build targets (all, android, desktop, embedded)' + required: true + default: 'all' + type: choice + options: + - all + - android + - desktop + - embedded + run_tests: + description: 'Run full test suite' + required: false + default: true + type: boolean + +env: + CARGO_TERM_COLOR: always + DEBIAN_FRONTEND: noninteractive + +jobs: + detect-changes: + runs-on: ubuntu-latest + outputs: + core-changed: ${{ steps.changes.outputs.core }} + android-changed: ${{ steps.changes.outputs.android }} + scripts-changed: ${{ steps.changes.outputs.scripts }} + docs-changed: ${{ steps.changes.outputs.docs }} + workflows-changed: ${{ steps.changes.outputs.workflows }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + core: + - 'src/**' + - 'meson.build' + - 'meson_options.txt' + - 'units/**' + android: + - 'android/**' + - 'scripts/android_**' + scripts: + - 'scripts/**' + docs: + - 'docs/**' + - '*.md' + workflows: + - '.github/**' + + matrix-build: + needs: detect-changes + if: > + needs.detect-changes.outputs.core-changed == 'true' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + include: + # Desktop Linux builds + - os: ubuntu-22.04 + arch: x86_64 + target: desktop + cc: gcc + build_type: release + - os: ubuntu-22.04 + arch: x86_64 + target: desktop + cc: clang + build_type: debug + + # ARM64/aarch64 builds + - os: ubuntu-22.04 + arch: aarch64 + target: embedded + cc: gcc + build_type: release + - os: ubuntu-22.04 + arch: aarch64 + target: android + cc: clang + build_type: release + # macOS builds for cross-platform validation + - os: macos-14 + arch: arm64 + target: embedded + cc: clang + build_type: release + + env: + CC: ${{ matrix.cc }} + ARCH: ${{ matrix.arch }} + BUILD_TYPE: ${{ matrix.build_type }} + TARGET: ${{ matrix.target }} + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Build Environment + run: | + case "${{ matrix.os }}" in + ubuntu-*) + sudo apt update + sudo apt install -y \ + gperf pkg-config ninja-build \ + libblkid-dev libudev-dev libmount-dev \ + libkmod-dev libcap-dev libdbus-1-dev \ + libacl1-dev libpam0g-dev libaudit-dev \ + libgcrypt20-dev libssl-dev libcryptsetup-dev \ + libgnutls28-dev libpwquality-dev \ + python3-pip python3-jinja2 python3-lxml + + # Install aarch64 cross-compilation tools if needed + if [ "${{ matrix.arch }}" = "aarch64" ]; then + sudo apt install -y gcc-aarch64-linux-gnu + export CC=aarch64-linux-gnu-gcc + fi + ;; + macos-*) + brew install meson ninja pkg-config gperf + ;; + esac + + - name: Setup Python Dependencies + run: | + pip3 install -r .github/workflows/requirements.txt + + - name: Configure Cross-Compilation + if: matrix.arch == 'aarch64' + run: | + cat > cross-aarch64.txt << EOF + [binaries] + c = 'aarch64-linux-gnu-gcc' + cpp = 'aarch64-linux-gnu-g++' + ar = 'aarch64-linux-gnu-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'pkg-config' + + [host_machine] + system = 'linux' + cpu_family = 'aarch64' + cpu = 'aarch64' + endian = 'little' + EOF + + - name: Setup Build Configuration + run: | + MESON_ARGS="-Dmode=release" + + case "${{ matrix.target }}" in + android) + MESON_ARGS="$MESON_ARGS -Dandroid=true -Dportable=true" + ;; + embedded) + MESON_ARGS="$MESON_ARGS -Dportable=true -Dsplit-bin=true" + ;; + desktop) + MESON_ARGS="$MESON_ARGS -Ddesktop=true" + ;; + esac + + if [ "${{ matrix.build_type }}" = "debug" ]; then + MESON_ARGS="$MESON_ARGS -Dmode=developer" + MESON_ARGS="$MESON_ARGS -Db_sanitize=address,undefined" + fi + + echo "MESON_ARGS=$MESON_ARGS" >> $GITHUB_ENV + + - name: Configure Build + run: | + CROSS_FILE="" + if [ "${{ matrix.arch }}" = "aarch64" ]; then + CROSS_FILE="--cross-file cross-aarch64.txt" + fi + + meson setup build $CROSS_FILE $MESON_ARGS + + - name: Build + run: | + meson compile -C build -j $(nproc) + + - name: Run Tests + if: > + github.event.inputs.run_tests != 'false' && + matrix.arch == 'x86_64' + run: | + meson test -C build --print-errorlogs + + - name: Package Build Artifacts + run: | + mkdir -p artifacts/${{ matrix.target }}-${{ matrix.arch }} + + # Copy binaries and libraries + find build -name "*.so*" -o -name "systemd*" -type f -executable | \ + while read file; do + cp "$file" artifacts/${{ matrix.target }}-${{ matrix.arch }}/ + done + + # Create manifest + cat > artifacts/${{ matrix.target }}-${{ matrix.arch }}/build-manifest.json << EOF + { + "target": "${{ matrix.target }}", + "arch": "${{ matrix.arch }}", + "build_type": "${{ matrix.build_type }}", + "compiler": "${{ matrix.cc }}", + "commit": "${{ github.sha }}", + "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "os": "${{ matrix.os }}" + } + EOF + + - name: Upload Build Artifacts + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.target }}-${{ matrix.arch }}-${{ matrix.cc }} + path: artifacts/ + retention-days: 30 + + android-build: + needs: detect-changes + if: > + needs.detect-changes.outputs.android-changed == 'true' || + github.event.inputs.build_targets == 'android' || + github.event.inputs.build_targets == 'all' + runs-on: ubuntu-latest + timeout-minutes: 45 + + strategy: + matrix: + api-level: [24, 28, 31, 34] + abi: [arm64-v8a, armeabi-v7a] + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Setup Android SDK + uses: android-actions/setup-android@v3 + with: + api-level: ${{ matrix.api-level }} + + - name: Create Android Project + run: | + # Create Android project structure if it doesn't exist + mkdir -p android/app/src/main/java/com/spiralgang/filesystemds + mkdir -p android/app/src/main/res/{layout,values} + + # Run the Android APK agent setup + if [ -f scripts/android_apk_agent.sh ]; then + chmod +x scripts/android_apk_agent.sh + scripts/android_apk_agent.sh setup-project + fi + + - name: Cache Gradle Dependencies + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + android/.gradle + key: > + ${{ runner.os }}-gradle-${{ + hashFiles('android/**/*.gradle*') + }} + + - name: Build APK + run: | + cd android + if [ ! -f gradlew ]; then + gradle wrapper + chmod +x gradlew + fi + ./gradlew assembleDebug + + - name: Upload APK + uses: actions/upload-artifact@v4 + with: + name: apk-api${{ matrix.api-level }}-${{ matrix.abi }} + path: android/app/build/outputs/apk/debug/*.apk + + security-scan: + runs-on: ubuntu-latest + needs: detect-changes + if: > + needs.detect-changes.outputs.core-changed == 'true' || + needs.detect-changes.outputs.scripts-changed == 'true' + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run Trivy Security Scan + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy Results + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' + + - name: Run Semgrep Security Scan + uses: semgrep/semgrep-action@v1 + with: + config: auto + + performance-test: + runs-on: ubuntu-latest + needs: matrix-build + if: > + github.event_name == 'schedule' || + github.event.inputs.run_tests == 'true' + + steps: + - uses: actions/checkout@v4 + + - name: Download Build Artifacts + uses: actions/download-artifact@v4 + with: + pattern: build-* + merge-multiple: true + + - name: Run Performance Benchmarks + run: | + # Run basic performance tests + if [ -f scripts/test_suite.sh ]; then + chmod +x scripts/test_suite.sh + timeout 300 scripts/test_suite.sh performance || \ + echo "Performance tests completed with warnings" + fi + + - name: Upload Performance Results + uses: actions/upload-artifact@v4 + with: + name: performance-results + path: | + **/performance-*.log + **/benchmark-*.json + + notification: + runs-on: ubuntu-latest + needs: [matrix-build, android-build, security-scan] + if: always() + + steps: + - name: Build Status Summary + run: | + echo "## 🏗️ Build Summary" >> $GITHUB_STEP_SUMMARY + echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Matrix Build | ${{ needs.matrix-build.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Android Build | ${{ needs.android-build.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Security Scan | ${{ needs.security-scan.result }} |" >> $GITHUB_STEP_SUMMARY + + if [ "${{ needs.matrix-build.result }}" = "success" ] && \ + [ "${{ needs.android-build.result }}" = "success" ]; then + echo "✅ All builds completed successfully!" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Some builds failed. Check individual job logs." >> $GITHUB_STEP_SUMMARY + fi \ No newline at end of file diff --git a/.github/workflows/dependency-security.yml b/.github/workflows/dependency-security.yml new file mode 100644 index 00000000000..26636a48f02 --- /dev/null +++ b/.github/workflows/dependency-security.yml @@ -0,0 +1,289 @@ +--- +name: Advanced Dependency and Security Management + +on: + schedule: + # Run weekly on Sundays at 3 AM UTC + - cron: '0 3 * * 0' + pull_request: + paths: + - '.github/workflows/requirements.txt' + - 'meson.build' + - 'android/app/build.gradle' + workflow_dispatch: + inputs: + scan_type: + description: 'Type of scan to perform' + required: true + default: 'all' + type: choice + options: + - all + - dependencies + - security + - licenses + +jobs: + dependency-audit: + runs-on: ubuntu-latest + outputs: + vulnerabilities-found: ${{ steps.audit.outputs.vulnerabilities }} + updates-available: ${{ steps.audit.outputs.updates }} + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Security Tools + run: | + pip install safety bandit semgrep pip-audit + + - name: Python Dependency Audit + id: audit + run: | + echo "## 🔍 Python Dependency Audit" >> $GITHUB_STEP_SUMMARY + + # Check for known vulnerabilities + if pip-audit --format=json --output=python-vulnerabilities.json; then + echo "✅ No Python vulnerabilities found" >> $GITHUB_STEP_SUMMARY + echo "vulnerabilities=false" >> $GITHUB_OUTPUT + else + echo "❌ Python vulnerabilities detected" >> $GITHUB_STEP_SUMMARY + echo "vulnerabilities=true" >> $GITHUB_OUTPUT + fi + + # Check for outdated packages + pip list --outdated --format=json > outdated-packages.json + OUTDATED_COUNT=$(jq length outdated-packages.json) + + if [ "$OUTDATED_COUNT" -gt 0 ]; then + echo "📦 $OUTDATED_COUNT outdated packages found" >> $GITHUB_STEP_SUMMARY + echo "updates=true" >> $GITHUB_OUTPUT + else + echo "✅ All packages up to date" >> $GITHUB_STEP_SUMMARY + echo "updates=false" >> $GITHUB_OUTPUT + fi + + - name: Upload Audit Results + uses: actions/upload-artifact@v4 + with: + name: dependency-audit-results + path: | + python-vulnerabilities.json + outdated-packages.json + + license-compliance: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: License Scan + uses: fossa-contrib/fossa-action@v2 + with: + api-key: ${{ secrets.FOSSA_API_KEY }} + + - name: Generate License Report + run: | + echo "## 📄 License Compliance Report" >> $GITHUB_STEP_SUMMARY + + # Check for license files + if find . -name "LICENSE*" -o -name "COPYING*" | grep -q .; then + echo "✅ License files found" >> $GITHUB_STEP_SUMMARY + else + echo "❌ No license files found" >> $GITHUB_STEP_SUMMARY + fi + + # List all license types + echo "### License Files:" >> $GITHUB_STEP_SUMMARY + find . -name "LICENSE*" -o -name "COPYING*" | while read file; do + echo "- $file" >> $GITHUB_STEP_SUMMARY + done + + codeql-security: + runs-on: ubuntu-latest + permissions: + security-events: write + + strategy: + matrix: + language: ['cpp', 'python'] + + steps: + - uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + - name: Install Dependencies + if: matrix.language == 'cpp' + run: | + sudo apt update + sudo apt install -y gperf pkg-config ninja-build meson \ + libblkid-dev libudev-dev libmount-dev libkmod-dev libcap-dev + + - name: Build for Analysis + if: matrix.language == 'cpp' + run: | + pip install -r .github/workflows/requirements.txt + meson setup build -Dmode=developer + meson compile -C build + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + + create-security-report: + runs-on: ubuntu-latest + needs: [dependency-audit, license-compliance, codeql-security] + if: always() + + steps: + - uses: actions/checkout@v4 + + - name: Download Audit Results + uses: actions/download-artifact@v4 + with: + name: dependency-audit-results + + - name: Generate Security Summary + run: | + echo "# 🛡️ Security and Compliance Report" > security-report.md + echo "Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> security-report.md + echo "" >> security-report.md + + echo "## 📊 Summary" >> security-report.md + echo "| Category | Status |" >> security-report.md + echo "|----------|--------|" >> security-report.md + echo "| Dependency Audit | ${{ needs.dependency-audit.result }} |" >> security-report.md + echo "| License Compliance | ${{ needs.license-compliance.result }} |" >> security-report.md + echo "| CodeQL Security | ${{ needs.codeql-security.result }} |" >> security-report.md + echo "" >> security-report.md + + if [ "${{ needs.dependency-audit.outputs.vulnerabilities-found }}" = "true" ]; then + echo "## ⚠️ Vulnerabilities Found" >> security-report.md + echo "Python dependency vulnerabilities detected. See audit results for details." >> security-report.md + echo "" >> security-report.md + fi + + if [ "${{ needs.dependency-audit.outputs.updates-available }}" = "true" ]; then + echo "## 📦 Package Updates Available" >> security-report.md + echo "Some packages have newer versions available." >> security-report.md + echo "" >> security-report.md + fi + + echo "## 🔗 Resources" >> security-report.md + echo "- [Security Policy](https://github.com/${{ github.repository }}/security/policy)" >> security-report.md + echo "- [Vulnerability Reporting](https://github.com/${{ github.repository }}/security/advisories)" >> security-report.md + + - name: Upload Security Report + uses: actions/upload-artifact@v4 + with: + name: security-report + path: security-report.md + + - name: Create Issue on Vulnerabilities + if: needs.dependency-audit.outputs.vulnerabilities-found == 'true' + uses: actions/github-script@v7 + with: + script: | + const title = '🚨 Security Vulnerabilities Detected'; + const body = `## Security Alert + + Automated security scan detected vulnerabilities in dependencies. + + **Details:** + - Scan Date: ${new Date().toISOString()} + - Commit: ${{ github.sha }} + - Workflow: ${{ github.workflow }} + + **Action Required:** + 1. Review the security report artifact + 2. Update vulnerable dependencies + 3. Re-run security scan to verify fixes + + **Files to Check:** + - \`.github/workflows/requirements.txt\` + - \`meson.build\` + - \`android/app/build.gradle\` + + This issue was created automatically. Please address the vulnerabilities promptly.`; + + // Check if issue already exists + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['security', 'automated'], + state: 'open' + }); + + const existingIssue = issues.data.find(issue => issue.title.includes('Security Vulnerabilities Detected')); + + if (!existingIssue) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['security', 'high-priority', 'automated'] + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existingIssue.number, + body: `## 🔄 Updated Security Scan Results\n\n${body}` + }); + } + + automated-dependency-update: + runs-on: ubuntu-latest + needs: dependency-audit + if: > + needs.dependency-audit.outputs.updates-available == 'true' && + github.event_name == 'schedule' + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Update Python Dependencies + run: | + # Update requirements file with latest versions + pip-compile --upgrade .github/workflows/requirements.in \ + --output-file .github/workflows/requirements.txt || true + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: 'chore: automated dependency updates' + title: '🤖 Automated Dependency Updates' + body: | + ## 🤖 Automated Dependency Updates + + This PR contains automated updates to project dependencies. + + **Changes:** + - Updated Python package versions in requirements.txt + - All updates passed security vulnerability checks + + **Review Instructions:** + 1. Verify all tests pass + 2. Check for any breaking changes in updated packages + 3. Ensure Android builds still work correctly + + This PR was created automatically by the dependency management workflow. + branch: automated/dependency-updates + delete-branch: true \ No newline at end of file diff --git a/.github/workflows/release-automation.yml b/.github/workflows/release-automation.yml new file mode 100644 index 00000000000..d0568dbab6a --- /dev/null +++ b/.github/workflows/release-automation.yml @@ -0,0 +1,448 @@ +--- +name: Automated Release Management + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + release_type: + description: 'Type of release' + required: true + default: 'patch' + type: choice + options: + - major + - minor + - patch + - prerelease + create_draft: + description: 'Create draft release' + required: false + default: true + type: boolean + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + prepare-release: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + changelog: ${{ steps.changelog.outputs.changelog }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Install Release Tools + run: | + npm install -g conventional-changelog-cli + pip install bump2version + + - name: Calculate Version + id: version + run: | + if [ "${{ github.event_name }}" = "push" ] && \ + [[ "${{ github.ref }}" =~ ^refs/tags/v.* ]]; then + # Extract version from tag + VERSION=${GITHUB_REF#refs/tags/v} + echo "version=$VERSION" >> $GITHUB_OUTPUT + else + # Calculate next version from current + CURRENT_VERSION=$(git describe --tags --abbrev=0 2>/dev/null | \ + sed 's/^v//' || echo "0.0.0") + + case "${{ github.event.inputs.release_type }}" in + major) + NEW_VERSION=$(echo $CURRENT_VERSION | \ + awk -F. '{print $1+1".0.0"}') + ;; + minor) + NEW_VERSION=$(echo $CURRENT_VERSION | \ + awk -F. '{print $1"."$2+1".0"}') + ;; + patch) + NEW_VERSION=$(echo $CURRENT_VERSION | \ + awk -F. '{print $1"."$2"."$3+1}') + ;; + prerelease) + NEW_VERSION="$CURRENT_VERSION-rc.$(date +%Y%m%d%H%M%S)" + ;; + esac + + echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT + fi + + - name: Generate Changelog + id: changelog + run: | + LAST_TAG=$(git describe --tags --abbrev=0 HEAD~1 2>/dev/null || echo "") + + if [ -n "$LAST_TAG" ]; then + CHANGELOG=$(git log $LAST_TAG..HEAD \ + --pretty=format:"- %s (%h)" --no-merges) + else + CHANGELOG=$(git log --pretty=format:"- %s (%h)" --no-merges) + fi + + # Save changelog to file for multiline output + echo "$CHANGELOG" > changelog.txt + echo "changelog<> $GITHUB_OUTPUT + echo "$CHANGELOG" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + build-release-artifacts: + needs: prepare-release + runs-on: ubuntu-latest + + strategy: + matrix: + include: + - target: linux-x86_64 + arch: x86_64 + - target: linux-aarch64 + arch: aarch64 + - target: android-arm64 + arch: arm64 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Build Environment + run: | + sudo apt update + sudo apt install -y \ + gperf pkg-config ninja-build \ + libblkid-dev libudev-dev libmount-dev \ + libkmod-dev libcap-dev libdbus-1-dev \ + python3-pip python3-jinja2 + + if [ "${{ matrix.arch }}" = "aarch64" ]; then + sudo apt install -y gcc-aarch64-linux-gnu + fi + + - name: Install Python Dependencies + run: pip install -r .github/workflows/requirements.txt + + - name: Configure Build + run: | + MESON_ARGS="-Dmode=release -Dportable=true" + CROSS_FILE="" + + if [ "${{ matrix.arch }}" = "aarch64" ]; then + cat > cross-aarch64.txt << EOF + [binaries] + c = 'aarch64-linux-gnu-gcc' + cpp = 'aarch64-linux-gnu-g++' + ar = 'aarch64-linux-gnu-ar' + strip = 'aarch64-linux-gnu-strip' + pkgconfig = 'pkg-config' + + [host_machine] + system = 'linux' + cpu_family = 'aarch64' + cpu = 'aarch64' + endian = 'little' + EOF + CROSS_FILE="--cross-file cross-aarch64.txt" + fi + + if [[ "${{ matrix.target }}" == *"android"* ]]; then + MESON_ARGS="$MESON_ARGS -Dandroid=true" + fi + + meson setup build $CROSS_FILE $MESON_ARGS + + - name: Build + run: meson compile -C build -j $(nproc) + + - name: Package Artifacts + run: | + VERSION="${{ needs.prepare-release.outputs.version }}" + PACKAGE_NAME="filesystemds-$VERSION-${{ matrix.target }}" + + mkdir -p "$PACKAGE_NAME"/{bin,lib,docs,scripts} + + # Copy binaries + find build -name "systemd*" -type f -executable \ + -exec cp {} "$PACKAGE_NAME/bin/" \; + + # Copy libraries + find build -name "*.so*" -type f -exec cp {} "$PACKAGE_NAME/lib/" \; + + # Copy documentation + cp README*.md LICENSE* "$PACKAGE_NAME/docs/" 2>/dev/null || true + + # Copy scripts + cp -r scripts/* "$PACKAGE_NAME/scripts/" 2>/dev/null || true + + # Create manifest + cat > "$PACKAGE_NAME/manifest.json" << EOF + { + "name": "FileSystemds", + "version": "$VERSION", + "target": "${{ matrix.target }}", + "arch": "${{ matrix.arch }}", + "build_date": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "commit": "${{ github.sha }}", + "components": [ + "systemd-core", + "mobile-platform", + "container-runtime", + "ai-core-manager" + ] + } + EOF + + # Create tarball + tar -czf "${PACKAGE_NAME}.tar.gz" "$PACKAGE_NAME" + + - name: Upload Release Artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.target }}-release + path: "*.tar.gz" + + build-android-release: + needs: prepare-release + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Setup JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Setup Android SDK + uses: android-actions/setup-android@v3 + + - name: Create Android Project + run: | + if [ ! -d "android" ]; then + # Use existing Android APK build setup + mkdir -p android + # The android-apk-build.yml workflow should handle this + fi + + - name: Update Version in Android + run: | + VERSION="${{ needs.prepare-release.outputs.version }}" + if [ -f "android/app/build.gradle" ]; then + sed -i "s/versionName \".*\"/versionName \"$VERSION\"/" \ + android/app/build.gradle + # Increment version code + CURRENT_CODE=$(grep versionCode android/app/build.gradle | \ + sed 's/.*versionCode \([0-9]*\).*/\1/') + NEW_CODE=$((CURRENT_CODE + 1)) + sed -i "s/versionCode $CURRENT_CODE/versionCode $NEW_CODE/" \ + android/app/build.gradle + fi + + - name: Build Release APK + run: | + cd android + if [ ! -f gradlew ]; then + gradle wrapper + chmod +x gradlew + fi + ./gradlew assembleRelease + + - name: Sign APK + if: env.ANDROID_SIGNING_KEY != '' + env: + ANDROID_SIGNING_KEY: ${{ secrets.ANDROID_SIGNING_KEY }} + ANDROID_SIGNING_PASSWORD: ${{ secrets.ANDROID_SIGNING_PASSWORD }} + run: | + # Sign the APK if signing credentials are available + echo "APK signing not implemented yet - using debug signing" + + - name: Upload APK + uses: actions/upload-artifact@v4 + with: + name: android-apk-release + path: android/app/build/outputs/apk/release/*.apk + + build-container-images: + needs: prepare-release + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Create Dockerfile + run: | + cat > Dockerfile << 'EOF' + FROM ubuntu:22.04 + + RUN apt-get update && apt-get install -y \ + gperf pkg-config ninja-build meson \ + libblkid-dev libudev-dev libmount-dev \ + libkmod-dev libcap-dev libdbus-1-dev \ + python3-pip python3-jinja2 python3-lxml \ + && rm -rf /var/lib/apt/lists/* + + WORKDIR /build + COPY . . + + RUN pip install -r .github/workflows/requirements.txt + RUN meson setup build -Dmode=release -Dportable=true + RUN meson compile -C build + + FROM ubuntu:22.04 + RUN apt-get update && apt-get install -y \ + libblkid1 libudev1 libmount1 libkmod2 \ + libcap2 libdbus-1-3 python3 \ + && rm -rf /var/lib/apt/lists/* + + COPY --from=0 /build/build/systemd* /usr/bin/ + COPY --from=0 /build/scripts /opt/filesystemds/scripts + COPY --from=0 /build/README*.md /opt/filesystemds/docs/ + + WORKDIR /opt/filesystemds + ENTRYPOINT ["/opt/filesystemds/scripts/platform_launcher.sh"] + EOF + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + create-release: + needs: [prepare-release, build-release-artifacts, build-android-release, build-container-images] + runs-on: ubuntu-latest + if: always() && needs.prepare-release.result == 'success' + + steps: + - uses: actions/checkout@v4 + + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + path: release-artifacts + + - name: Prepare Release Assets + run: | + cd release-artifacts + find . -name "*.tar.gz" -o -name "*.apk" | while read file; do + cp "$file" "../$(basename "$file")" + done + + - name: Create Release + uses: softprops/action-gh-release@v1 + id: create_release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ needs.prepare-release.outputs.version }} + name: > + FileSystemds v${{ needs.prepare-release.outputs.version }} + body: | + ## 🚀 FileSystemds v${{ needs.prepare-release.outputs.version }} + + ### 📱 Quick Download + + **Android APK:** [filesystemds-mobile-v${{ needs.prepare-release.outputs.version }}.apk](https://github.com/${{ github.repository }}/releases/download/v${{ needs.prepare-release.outputs.version }}/app-release.apk) + + ### 📋 What's Changed + + ${{ needs.prepare-release.outputs.changelog }} + + ### 📦 Available Downloads + + - **Android APK** - Mobile application for Android devices + - **Linux x86_64** - Desktop/server deployment package + - **Linux ARM64** - Embedded systems and ARM-based servers + - **Container Images** - Docker/Podman containers available on GHCR + + ### 🔧 Installation + + #### Android + 1. Download the APK file + 2. Enable "Unknown Sources" in Android settings + 3. Install the APK + + #### Linux + 1. Download the appropriate tar.gz for your architecture + 2. Extract: `tar -xzf filesystemds-v${{ needs.prepare-release.outputs.version }}-linux-*.tar.gz` + 3. Run: `./filesystemds-*/scripts/platform_launcher.sh init` + + #### Container + ```bash + docker run --rm -it ghcr.io/${{ github.repository }}:${{ needs.prepare-release.outputs.version }} + ``` + + ### 🛡️ Security + + All release artifacts are built from the same source code and have passed: + - Automated security scanning + - Dependency vulnerability checks + - Code quality analysis + - Cross-platform compatibility testing + + For security issues, please see our [Security Policy](https://github.com/${{ github.repository }}/security/policy). + + --- + + **Full Changelog**: https://github.com/${{ github.repository }}/compare/v${{ needs.prepare-release.outputs.version }}...v${{ needs.prepare-release.outputs.version }} + draft: ${{ github.event.inputs.create_draft == 'true' }} + prerelease: > + ${{ contains(needs.prepare-release.outputs.version, 'rc') || + contains(needs.prepare-release.outputs.version, 'beta') || + contains(needs.prepare-release.outputs.version, 'alpha') }} + + - name: Upload Release Assets + run: | + for file in *.tar.gz *.apk; do + if [ -f "$file" ]; then + gh release upload v${{ needs.prepare-release.outputs.version }} \ + "$file" + fi + done + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Update Latest Release Badge + if: github.event_name == 'push' + run: | + echo "Release v${{ needs.prepare-release.outputs.version }} created successfully!" + echo "## 🎉 Release Published" >> $GITHUB_STEP_SUMMARY + echo "Version: v${{ needs.prepare-release.outputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "Download: [GitHub Releases](https://github.com/${{ github.repository }}/releases/latest)" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.gitignore b/.gitignore index 460ed57cce8..ee74b37c7be 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,73 @@ __pycache__/ # Platform operations directory platform_ops/ + +# Modern Mobile Development +android/.gradle/ +android/build/ +android/app/build/ +android/app/src/main/gen/ +android/local.properties +android/captures/ +android/keystore.properties +*.apk +*.aab +*.ap_ +*.dex + +# Android Studio +.idea/ +.gradle/ +gradle.properties + +# Mobile Development +node_modules/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Asset Management (pointer-first) +*.model +*.weights +*.dataset +large_assets/ +cached_models/ + +# Cross-compilation +cross-*.txt +sysroot/ +toolchain/ + +# Container Images +*.tar +docker-compose.override.yml + +# AI Core temporary files +ai_cache/ +model_cache/ +inference_logs/ + +# Security keys and secrets +*.key +*.pem +*.p12 +secrets/ +.env.local +.env.production + +# Performance monitoring +*.prof +*.trace +benchmarks/ + +# IDE and editor files +.vscode/ +.kate-swp +*.sw? +cscope.* +GPATH +GRTAGS +GTAGS +*.orig +*.rej +.dir-locals.el diff --git a/README.md b/README.md index ad4a7413ff2..6c6618b1b1d 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,40 @@ +# FileSystemds +[![APK Build Status](https://github.com/spiralgang/FileSystemds/actions/workflows/android-apk-build.yml/badge.svg)](https://github.com/spiralgang/FileSystemds/actions/workflows/android-apk-build.yml) +[![Latest Release](https://img.shields.io/github/v/release/spiralgang/FileSystemds?include_prereleases)](https://github.com/spiralgang/FileSystemds/releases) +[![License](https://img.shields.io/github/license/spiralgang/FileSystemds)](https://github.com/spiralgang/FileSystemds/blob/main/LICENSE.GPL2) -### Lagacy README.md》》》》》 +**FileSystemds Mobile Platform** - A modular, AI-driven orchestration system designed for modern mobile/cloud-first environments. -System and Service Manager +## 📱 Quick Start -[![OBS Packages Status](https://build.opensuse.org/projects/system:systemd/packages/systemd/badge.svg?type=default)](https://build.opensuse.org/project/show/system:systemd)
-[![Semaphore CI 2.0 Build Status](https://the-real-systemd.semaphoreci.com/badges/systemd/branches/main.svg?style=shields)](https://the-real-systemd.semaphoreci.com/projects/systemd)
-[![Coverity Scan Status](https://scan.coverity.com/projects/350/badge.svg)](https://scan.coverity.com/projects/systemd)
-[![OSS-Fuzz Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/systemd.svg)](https://oss-fuzz-build-logs.storage.googleapis.com/index.html#systemd)
-[![CIFuzz](https://github.com/systemd/systemd/actions/workflows/cifuzz.yml/badge.svg)](https://github.com/systemd/systemd/actions/workflows/cifuzz.yml)
-[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1369/badge)](https://bestpractices.coreinfrastructure.org/projects/1369)
-[![Fossies codespell report](https://fossies.org/linux/test/systemd-main.tar.gz/codespell.svg)](https://fossies.org/linux/test/systemd-main.tar.gz/codespell.html)
-[![Translation status](https://translate.fedoraproject.org/widget/systemd/svg-badge.svg)](https://translate.fedoraproject.org/engage/systemd/)
-[![Coverage Status](https://coveralls.io/repos/github/systemd/systemd/badge.svg?branch=main)](https://coveralls.io/github/systemd/systemd?branch=main)
-[![Packaging status](https://repology.org/badge/tiny-repos/systemd.svg)](https://repology.org/project/systemd/versions)
-[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/systemd/systemd/badge)](https://securityscorecards.dev/viewer/?platform=github.com&org=systemd&repo=systemd) +For complete information about the FileSystemds Mobile Platform, including download links and installation instructions, please see: -## Details +**➡️ [README_MOBILE.md](README_MOBILE.md) - Complete Mobile Platform Documentation** -Most documentation is available on [systemd's web site](https://systemd.io/). +## 🏗️ Architecture -Assorted, older, general information about systemd can be found in the [systemd Wiki](https://www.freedesktop.org/wiki/Software/systemd). +FileSystemds is transitioning from legacy foundations to a modern, modular architecture: -Information about build requirements is provided in the [README file](README). +- **Modular, composable components** - Replaceable modules instead of hardwired logic +- **Agent-driven workflows** - Event-driven, API-first automation +- **Mobile/cloud/edge ready** - Cross-platform, stateless design +- **Pointer-first artifact management** - Secure, audited asset handling -Consult our [NEWS file](NEWS) for information about what's new in the most recent systemd versions. +## 📖 Documentation -Please see the [Code Map](docs/ARCHITECTURE.md) for information about this repository's layout and content. +- [Mobile Platform Guide](README_MOBILE.md) - Primary documentation +- [Contributing Guidelines](docs/CONTRIBUTING.md) - How to contribute +- [Code of Conduct](docs/CODE_OF_CONDUCT.md) - Community guidelines +- [Project Architecture](docs/ARCHITECTURE.md) - Technical overview -Please see the [Hacking guide](docs/HACKING.md) for information on how to hack on systemd and test your modifications. +## 🤝 Contributing -Please see our [Contribution Guidelines](docs/CONTRIBUTING.md) for more information about filing GitHub Issues and posting GitHub Pull Requests. +We welcome contributions! Please see our [Contributing Guidelines](docs/CONTRIBUTING.md) for details on how to get started. -When preparing patches for systemd, please follow our [Coding Style Guidelines](docs/CODING_STYLE.md). +## 📄 License -If you are looking for support, please contact our [mailing list](https://lists.freedesktop.org/mailman/listinfo/systemd-devel), join our [IRC channel #systemd on libera.chat](https://web.libera.chat/#systemd) or [Matrix channel](https://matrix.to/#/#systemd-project:matrix.org) - -Stable branches with backported patches are available in the [stable repo](https://github.com/systemd/systemd-stable). - -We have a security bug bounty program sponsored by the [Sovereign Tech Fund](https://www.sovereigntechfund.de/) hosted on [YesWeHack](https://yeswehack.com/programs/systemd-bug-bounty-program) - -Repositories with distribution packages built from git main are [available on OBS](https://software.opensuse.org//download.html?project=system%3Asystemd&package=systemd) +This project is licensed under multiple licenses. See the [LICENSE files](LICENSES/) for details. diff --git a/android/app/build.gradle b/android/app/build.gradle new file mode 100644 index 00000000000..dcd144707d2 --- /dev/null +++ b/android/app/build.gradle @@ -0,0 +1,212 @@ +plugins { + id 'com.android.application' + id 'org.jetbrains.kotlin.android' + id 'kotlin-parcelize' +} + +android { + namespace 'com.spiralgang.filesystemds' + compileSdk rootProject.ext.compileSdkVersion + + defaultConfig { + applicationId "com.spiralgang.filesystemds" + minSdk rootProject.ext.minSdkVersion + targetSdk rootProject.ext.targetSdkVersion + versionCode 1 + versionName "1.0.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + + // Enable vector drawables support + vectorDrawables.useSupportLibrary = true + + // Build config fields + buildConfigField "String", "BUILD_TIME", "\"${new Date().format('yyyy-MM-dd HH:mm:ss')}\"" + buildConfigField "String", "GIT_COMMIT", "\"${getGitCommitHash()}\"" + + // NDK configuration for native libraries + ndk { + abiFilters 'arm64-v8a', 'armeabi-v7a', 'x86_64' + } + } + + buildTypes { + debug { + debuggable true + applicationIdSuffix ".debug" + versionNameSuffix "-debug" + + // Enable code coverage + testCoverageEnabled true + + // Proguard for debug builds (optional) + minifyEnabled false + shrinkResources false + + buildConfigField "boolean", "DEBUG_MODE", "true" + buildConfigField "String", "API_BASE_URL", "\"https://api-dev.filesystemds.com\"" + } + + release { + minifyEnabled true + shrinkResources true + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + + // Signing config (will be configured in CI/CD) + signingConfig signingConfigs.debug + + buildConfigField "boolean", "DEBUG_MODE", "false" + buildConfigField "String", "API_BASE_URL", "\"https://api.filesystemds.com\"" + } + + benchmark { + initWith release + matchingFallbacks = ['release'] + debuggable false + signingConfig signingConfigs.debug + } + } + + compileOptions { + sourceCompatibility JavaVersion.VERSION_17 + targetCompatibility JavaVersion.VERSION_17 + + // Enable core library desugaring for modern APIs + coreLibraryDesugaringEnabled true + } + + kotlinOptions { + jvmTarget = '17' + + // Enable Kotlin compiler optimizations + freeCompilerArgs += [ + '-opt-in=kotlin.RequiresOptIn', + '-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi' + ] + } + + buildFeatures { + buildConfig true + compose true + viewBinding true + } + + composeOptions { + kotlinCompilerExtensionVersion rootProject.ext.composeCompilerVersion + } + + packaging { + resources { + excludes += [ + '/META-INF/{AL2.0,LGPL2.1}', + '/META-INF/gradle/incremental.annotation.processors' + ] + } + } + + // Lint configuration + lint { + checkReleaseBuilds false + abortOnError false + disable 'MissingTranslation' + } + + // Test options + testOptions { + unitTests { + includeAndroidResources = true + returnDefaultValues = true + } + + animationsDisabled = true + } +} + +dependencies { + // Core Android dependencies + implementation "androidx.core:core-ktx:${rootProject.ext.coreKtxVersion}" + implementation "androidx.appcompat:appcompat:${rootProject.ext.appCompatVersion}" + implementation "com.google.android.material:material:${rootProject.ext.materialVersion}" + implementation "androidx.constraintlayout:constraintlayout:${rootProject.ext.constraintLayoutVersion}" + + // Lifecycle components + implementation "androidx.lifecycle:lifecycle-runtime-ktx:${rootProject.ext.lifecycleVersion}" + implementation "androidx.lifecycle:lifecycle-viewmodel-ktx:${rootProject.ext.lifecycleVersion}" + implementation "androidx.lifecycle:lifecycle-livedata-ktx:${rootProject.ext.lifecycleVersion}" + + // Activity & Fragment + implementation "androidx.activity:activity-ktx:${rootProject.ext.activityVersion}" + implementation "androidx.fragment:fragment-ktx:${rootProject.ext.fragmentVersion}" + + // Jetpack Compose + implementation platform("androidx.compose:compose-bom:${rootProject.ext.compose_version}") + implementation 'androidx.compose.ui:ui' + implementation 'androidx.compose.ui:ui-graphics' + implementation 'androidx.compose.ui:ui-tooling-preview' + implementation 'androidx.compose.material3:material3' + implementation 'androidx.activity:activity-compose' + implementation 'androidx.lifecycle:lifecycle-viewmodel-compose' + + // Navigation + implementation 'androidx.navigation:navigation-fragment-ktx:2.7.6' + implementation 'androidx.navigation:navigation-ui-ktx:2.7.6' + implementation 'androidx.navigation:navigation-compose:2.7.6' + + // Network & Serialization + implementation "com.squareup.retrofit2:retrofit:${rootProject.ext.retrofitVersion}" + implementation "com.squareup.retrofit2:converter-gson:${rootProject.ext.retrofitVersion}" + implementation "com.squareup.okhttp3:okhttp:${rootProject.ext.okhttpVersion}" + implementation "com.squareup.okhttp3:logging-interceptor:${rootProject.ext.okhttpVersion}" + implementation "com.google.code.gson:gson:${rootProject.ext.gsonVersion}" + + // Coroutines + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3' + + // Security + implementation "androidx.security:security-crypto:${rootProject.ext.securityCryptoVersion}" + + // Permissions + implementation 'com.karumi:dexter:6.2.3' + + // Image loading + implementation 'com.github.bumptech.glide:glide:4.16.0' + + // FileSystemds specific dependencies + implementation 'androidx.work:work-runtime-ktx:2.9.0' + implementation 'androidx.room:room-runtime:2.6.1' + implementation 'androidx.room:room-ktx:2.6.1' + + // Core library desugaring + coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:2.0.4' + + // Development tools + debugImplementation 'com.squareup.leakcanary:leakcanary-android:2.12' + debugImplementation 'androidx.compose.ui:ui-tooling' + debugImplementation 'androidx.compose.ui:ui-test-manifest' + + // Testing dependencies + testImplementation "junit:junit:${rootProject.ext.junitVersion}" + testImplementation 'org.mockito:mockito-core:5.8.0' + testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3' + testImplementation 'androidx.arch.core:core-testing:2.2.0' + + androidTestImplementation "androidx.test.ext:junit:${rootProject.ext.extJunitVersion}" + androidTestImplementation "androidx.test.espresso:espresso-core:${rootProject.ext.espressoVersion}" + androidTestImplementation 'androidx.compose.ui:ui-test-junit4' + androidTestImplementation 'androidx.test:runner:1.5.2' + androidTestImplementation 'androidx.test:rules:1.5.0' +} + +// Helper function to get git commit hash +def getGitCommitHash() { + try { + def stdout = new ByteArrayOutputStream() + exec { + commandLine 'git', 'rev-parse', '--short', 'HEAD' + standardOutput = stdout + } + return stdout.toString().trim() + } catch (Exception e) { + return "unknown" + } +} \ No newline at end of file diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..4565ae8ded7 --- /dev/null +++ b/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/android/app/src/main/java/com/spiralgang/filesystemds/MainActivity.kt b/android/app/src/main/java/com/spiralgang/filesystemds/MainActivity.kt new file mode 100644 index 00000000000..6003c183981 --- /dev/null +++ b/android/app/src/main/java/com/spiralgang/filesystemds/MainActivity.kt @@ -0,0 +1,254 @@ +package com.spiralgang.filesystemds + +import android.Manifest +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Bundle +import android.widget.Toast +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.layout.* +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material3.* +import androidx.compose.runtime.* +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.core.content.ContextCompat +import androidx.lifecycle.viewmodel.compose.viewModel +import kotlinx.coroutines.launch + +class MainActivity : ComponentActivity() { + + private val requestPermissionLauncher = registerForActivityResult( + ActivityResultContracts.RequestPermission() + ) { isGranted: Boolean -> + if (isGranted) { + initializePlatform() + } else { + Toast.makeText(this, "Storage permission required for FileSystemds", Toast.LENGTH_LONG).show() + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + checkPermissionsAndInitialize() + + setContent { + FileSystemdsTheme { + Surface( + modifier = Modifier.fillMaxSize(), + color = MaterialTheme.colorScheme.background + ) { + MainScreen() + } + } + } + } + + private fun checkPermissionsAndInitialize() { + when { + ContextCompat.checkSelfPermission( + this, + Manifest.permission.WRITE_EXTERNAL_STORAGE + ) == PackageManager.PERMISSION_GRANTED -> { + initializePlatform() + } + else -> { + requestPermissionLauncher.launch(Manifest.permission.WRITE_EXTERNAL_STORAGE) + } + } + } + + private fun initializePlatform() { + // Initialize platform services + Toast.makeText(this, "FileSystemds Platform Initialized", Toast.LENGTH_SHORT).show() + } +} + +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun MainScreen() { + val context = LocalContext.current + + Column( + modifier = Modifier + .fillMaxSize() + .padding(16.dp) + ) { + // Header + Card( + modifier = Modifier + .fillMaxWidth() + .padding(bottom = 16.dp), + elevation = CardDefaults.cardElevation(defaultElevation = 4.dp) + ) { + Column( + modifier = Modifier.padding(16.dp), + horizontalAlignment = Alignment.CenterHorizontally + ) { + Text( + text = "FileSystemds Mobile Platform", + style = MaterialTheme.typography.headlineMedium, + fontWeight = FontWeight.Bold + ) + Text( + text = "Version: ${BuildConfig.VERSION_NAME}", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + Text( + text = "Build: ${BuildConfig.GIT_COMMIT}", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + } + } + + // System Status + SystemStatusCard() + + Spacer(modifier = Modifier.height(16.dp)) + + // Platform Features + Text( + text = "Platform Features", + style = MaterialTheme.typography.headlineSmall, + fontWeight = FontWeight.Bold, + modifier = Modifier.padding(bottom = 8.dp) + ) + + val features = listOf( + "AI Core Manager" to "Intelligent system automation", + "Container Runtime" to "Chisel containers and QEMU VMs", + "Network Configuration" to "Advanced networking setup", + "Security Framework" to "Zero-trust security monitoring", + "Asset Manager" to "Resource and dependency management", + "Plugin System" to "Extensible architecture" + ) + + LazyColumn( + verticalArrangement = Arrangement.spacedBy(8.dp) + ) { + items(features) { (name, description) -> + FeatureCard(name = name, description = description) + } + } + + Spacer(modifier = Modifier.height(16.dp)) + + // Action Buttons + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceEvenly + ) { + Button( + onClick = { + Toast.makeText(context, "Running diagnostics...", Toast.LENGTH_SHORT).show() + }, + modifier = Modifier.weight(1f).padding(end = 8.dp) + ) { + Text("Run Diagnostics") + } + + Button( + onClick = { + Toast.makeText(context, "Settings coming soon...", Toast.LENGTH_SHORT).show() + }, + modifier = Modifier.weight(1f).padding(start = 8.dp) + ) { + Text("Settings") + } + } + } +} + +@Composable +fun SystemStatusCard() { + Card( + modifier = Modifier.fillMaxWidth(), + elevation = CardDefaults.cardElevation(defaultElevation = 2.dp) + ) { + Column( + modifier = Modifier.padding(16.dp) + ) { + Text( + text = "System Status", + style = MaterialTheme.typography.titleMedium, + fontWeight = FontWeight.Bold + ) + + Spacer(modifier = Modifier.height(8.dp)) + + StatusRow("Platform", "Running") + StatusRow("AI Core", "Active") + StatusRow("Container Runtime", "Ready") + StatusRow("Network", "Configured") + + Spacer(modifier = Modifier.height(8.dp)) + + Text( + text = "Uptime: ${java.text.SimpleDateFormat("HH:mm:ss").format(java.util.Date())}", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + } + } +} + +@Composable +fun StatusRow(label: String, status: String) { + Row( + modifier = Modifier + .fillMaxWidth() + .padding(vertical = 2.dp), + horizontalArrangement = Arrangement.SpaceBetween + ) { + Text( + text = label, + style = MaterialTheme.typography.bodyMedium + ) + Text( + text = status, + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.primary + ) + } +} + +@Composable +fun FeatureCard(name: String, description: String) { + Card( + modifier = Modifier.fillMaxWidth(), + elevation = CardDefaults.cardElevation(defaultElevation = 1.dp) + ) { + Column( + modifier = Modifier.padding(16.dp) + ) { + Text( + text = name, + style = MaterialTheme.typography.titleSmall, + fontWeight = FontWeight.Bold + ) + Text( + text = description, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + } + } +} + +@Preview(showBackground = true) +@Composable +fun MainScreenPreview() { + FileSystemdsTheme { + MainScreen() + } +} \ No newline at end of file diff --git a/android/app/src/main/java/com/spiralgang/filesystemds/ui/theme/Theme.kt b/android/app/src/main/java/com/spiralgang/filesystemds/ui/theme/Theme.kt new file mode 100644 index 00000000000..b0c6cfab191 --- /dev/null +++ b/android/app/src/main/java/com/spiralgang/filesystemds/ui/theme/Theme.kt @@ -0,0 +1,54 @@ +package com.spiralgang.filesystemds.ui.theme + +import androidx.compose.foundation.isSystemInDarkTheme +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.darkColorScheme +import androidx.compose.material3.lightColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.ui.graphics.Color + +private val DarkColorScheme = darkColorScheme( + primary = Color(0xFF6366F1), + secondary = Color(0xFF06B6D4), + tertiary = Color(0xFF10B981), + background = Color(0xFF0F172A), + surface = Color(0xFF1E293B), + error = Color(0xFFEF4444), + onPrimary = Color.White, + onSecondary = Color.White, + onTertiary = Color.White, + onBackground = Color(0xFFE2E8F0), + onSurface = Color(0xFFE2E8F0), + onError = Color.White +) + +private val LightColorScheme = lightColorScheme( + primary = Color(0xFF4F46E5), + secondary = Color(0xFF0891B2), + tertiary = Color(0xFF059669), + background = Color(0xFFFAFAFA), + surface = Color.White, + error = Color(0xFFDC2626), + onPrimary = Color.White, + onSecondary = Color.White, + onTertiary = Color.White, + onBackground = Color(0xFF1F2937), + onSurface = Color(0xFF1F2937), + onError = Color.White +) + +@Composable +fun FileSystemdsTheme( + darkTheme: Boolean = isSystemInDarkTheme(), + content: @Composable () -> Unit +) { + val colorScheme = when { + darkTheme -> DarkColorScheme + else -> LightColorScheme + } + + MaterialTheme( + colorScheme = colorScheme, + content = content + ) +} \ No newline at end of file diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml new file mode 100644 index 00000000000..2c4c80812b4 --- /dev/null +++ b/android/app/src/main/res/values/strings.xml @@ -0,0 +1,61 @@ + + FileSystemds + Mobile Platform for Advanced System Management + + + Platform Status + System Running + AI Core Active + Container Runtime Ready + Network Configured + + + Run Diagnostics + Settings + Start + Stop + Restart + + + AI Core Manager + Intelligent system automation and decision making + Platform Launcher + Central orchestration and lifecycle management + Container Runtime + Chisel containers and QEMU virtual machines + Network Manager + Advanced networking configuration + Security Framework + Integrity monitoring and verification + Asset Manager + Resource and asset management + + + FileSystemds Platform Service Running + AI Core Service Active + Platform Initialized Successfully + Running System Diagnostics + + + Storage permission required for FileSystemds operation + Permission granted + Permission denied + + + Platform initialization failed + Failed to start service + Network configuration error + Container runtime error + + + Advanced Settings + Platform Configuration + AI Core Settings + Network Settings + Security Settings + + + Version Information + Build Information + © 2024 SpiralGang - FileSystemds Mobile Platform + \ No newline at end of file diff --git a/android/build.gradle b/android/build.gradle new file mode 100644 index 00000000000..ac5689d73d7 --- /dev/null +++ b/android/build.gradle @@ -0,0 +1,60 @@ +buildscript { + ext.kotlin_version = "1.9.21" + ext.compose_version = "2024.02.00" + dependencies { + classpath 'com.android.tools.build:gradle:8.2.2' + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" + classpath 'com.google.gms:google-services:4.4.0' + } +} + +plugins { + id 'com.github.ben-manes.versions' version '0.50.0' +} + +allprojects { + repositories { + google() + mavenCentral() + maven { url 'https://jitpack.io' } + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} + +// Dependency version management +ext { + minSdkVersion = 24 + compileSdkVersion = 34 + targetSdkVersion = 34 + + // AndroidX versions + coreKtxVersion = "1.12.0" + appCompatVersion = "1.6.1" + materialVersion = "1.11.0" + constraintLayoutVersion = "2.1.4" + lifecycleVersion = "2.7.0" + activityVersion = "1.8.2" + fragmentVersion = "1.6.2" + + // Compose versions + composeCompilerVersion = "1.5.8" + + // Network & Serialization + retrofitVersion = "2.9.0" + okhttpVersion = "4.12.0" + gsonVersion = "2.10.1" + + // Testing versions + junitVersion = "4.13.2" + extJunitVersion = "1.1.5" + espressoVersion = "3.5.1" + + // Security & Encryption + securityCryptoVersion = "1.0.0" + + // Performance & Monitoring + leakCanaryVersion = "2.12" +} \ No newline at end of file diff --git a/cross-aarch64.txt.template b/cross-aarch64.txt.template new file mode 100644 index 00000000000..dee39a08c96 --- /dev/null +++ b/cross-aarch64.txt.template @@ -0,0 +1,28 @@ +# FileSystemds Cross-Compilation Configuration Template +# This file is used to configure cross-compilation for ARM64/aarch64 targets + +[binaries] +c = 'aarch64-linux-gnu-gcc' +cpp = 'aarch64-linux-gnu-g++' +ar = 'aarch64-linux-gnu-ar' +strip = 'aarch64-linux-gnu-strip' +objcopy = 'aarch64-linux-gnu-objcopy' +objdump = 'aarch64-linux-gnu-objdump' +pkgconfig = 'pkg-config' +pkg-config = 'pkg-config' + +[built-in options] +c_args = ['-march=armv8-a', '-mtune=cortex-a72'] +cpp_args = ['-march=armv8-a', '-mtune=cortex-a72'] +c_link_args = [] +cpp_link_args = [] + +[properties] +# Additional properties for cross-compilation +needs_exe_wrapper = true +has_function_printf = true +[host_machine] +system = 'linux' +cpu_family = 'aarch64' +cpu = 'armv8' +endian = 'little' \ No newline at end of file diff --git a/docs/AUTOMATIC_BOOT_ASSESSMENT.md b/docs/AUTOMATIC_BOOT_ASSESSMENT.md deleted file mode 100644 index 2fbf86ec6a6..00000000000 --- a/docs/AUTOMATIC_BOOT_ASSESSMENT.md +++ /dev/null @@ -1,219 +0,0 @@ ---- -title: Automatic Boot Assessment -category: Booting -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Automatic Boot Assessment - -systemd provides support for automatically reverting back to the previous -version of the OS or kernel in case the system consistently fails to boot. The -[Boot Loader Specification](https://uapi-group.org/specifications/specs/boot_loader_specification/#boot-counting) -describes how to annotate boot loader entries with a counter that specifies how -many attempts should be made to boot it. This document describes how systemd -implements this scheme. - -The many different components involved in the implementation may be used -independently and in combination with other software to, for example, support -other boot loaders or take actions outside of the boot loader. - -Here's a brief overview of the complete set of components: - -* The - [`kernel-install(8)`](https://www.freedesktop.org/software/systemd/man/kernel-install.html) - script can optionally create boot loader entries that carry an initial boot - counter (the initial counter is configurable in `/etc/kernel/tries`). - -* The - [`systemd-boot(7)`](https://www.freedesktop.org/software/systemd/man/systemd-boot.html) - boot loader optionally maintains a per-boot-loader-entry counter described by - the [Boot Loader Specification](https://uapi-group.org/specifications/specs/boot_loader_specification/#boot-counting) - that is decreased by one on each attempt to boot the entry, prioritizing - entries that have non-zero counters over those which already reached a - counter of zero when choosing the entry to boot. - -* The `boot-complete.target` target unit (see - [`systemd.special(7)`](https://www.freedesktop.org/software/systemd/man/systemd.special.html)) - serves as a generic extension point both for units that are necessary to - consider a boot successful (e.g. `systemd-boot-check-no-failures.service` - described below), and units that want to act only if the boot is - successful (e.g. `systemd-bless-boot.service` described below). - -* The - [`systemd-boot-check-no-failures.service(8)`](https://www.freedesktop.org/software/systemd/man/systemd-boot-check-no-failures.service.html) - service is a simple service health check tool. When enabled it becomes an - indirect dependency of `systemd-bless-boot.service` (by means of - `boot-complete.target`, see below), ensuring that the boot will not be - considered successful if there are any failed services. - -* The - [`systemd-bless-boot.service(8)`](https://www.freedesktop.org/software/systemd/man/systemd-bless-boot.service.html) - service automatically marks a boot loader entry, for which boot counting as - mentioned above is enabled, as "good" when a boot has been determined to be - successful, thus turning off boot counting for it. - -* The - [`systemd-bless-boot-generator(8)`](https://www.freedesktop.org/software/systemd/man/systemd-bless-boot-generator.html) - generator automatically pulls in `systemd-bless-boot.service` when use of - `systemd-boot` with boot counting enabled is detected. - -## Details - -As described in the -[Boot Loader Specification](https://uapi-group.org/specifications/specs/boot_loader_specification/#boot-counting), -the boot counting data is stored in the file name of the boot loader entries as -a plus (`+`), followed by a number, optionally followed by `-` and another -number, right before the file name suffix (`.conf` or `.efi`). - -The first number is the "tries left" counter encoding how many attempts to boot -this entry shall still be made. The second number is the "tries done" counter, -encoding how many failed attempts to boot it have already been made. Each time -a boot loader entry marked this way is booted the first counter is decremented, -and the second one incremented. (If the second counter is missing, then it is -assumed to be equivalent to zero.) If the boot attempt completed successfully -the entry's counters are removed from the name (entry state "good"), thus -turning off boot counting for the future. - -## Walkthrough - -Here's an example walkthrough of how this all fits together. - -1. The user runs `echo 3 >/etc/kernel/tries` to enable boot counting. - -2. A new kernel is installed. `kernel-install` is used to generate a new boot - loader entry file for it. Let's say the version string for the new kernel is - `4.14.11-300.fc27.x86_64`, a new boot loader entry - `/boot/loader/entries/4.14.11-300.fc27.x86_64+3.conf` is hence created. - -3. The system is booted for the first time after the new kernel has been - installed. The boot loader now sees the `+3` counter in the entry file - name. It hence renames the file to `4.14.11-300.fc27.x86_64+2-1.conf` - indicating that at this point one attempt has started. - After the rename completed, the entry is booted as usual. - -4. Let's say this attempt to boot fails. On the following boot the boot loader - will hence see the `+2-1` tag in the name, and will hence rename the entry file to - `4.14.11-300.fc27.x86_64+1-2.conf`, and boot it. - -5. Let's say the boot fails again. On the subsequent boot the loader will hence - see the `+1-2` tag, and rename the file to - `4.14.11-300.fc27.x86_64+0-3.conf` and boot it. - -6. If this boot also fails, on the next boot the boot loader will see the tag - `+0-3`, i.e. the counter reached zero. At this point the entry will be - considered "bad", and ordered after all non-bad entries. The next newest - boot entry is now tried, i.e. the system automatically reverted to an - earlier version. - -The above describes the walkthrough when the selected boot entry continuously -fails. Let's have a look at an alternative ending to this walkthrough. In this -scenario the first 4 steps are the same as above: - -1. *as above* - -2. *as above* - -3. *as above* - -4. *as above* - -5. Let's say the second boot succeeds. The kernel initializes properly, systemd - is started and invokes all generators. - -6. One of the generators started is `systemd-bless-boot-generator` which - detects that boot counting is used. It hence pulls - `systemd-bless-boot.service` into the initial transaction. - -7. `systemd-bless-boot.service` is ordered after and `Requires=` the generic - `boot-complete.target` unit. This unit is hence also pulled into the initial - transaction. - -8. The `boot-complete.target` unit is ordered after and pulls in various units - that are required to succeed for the boot process to be considered - successful. One such unit is `systemd-boot-check-no-failures.service`. - -9. The graphical desktop environment installed on the machine starts a - service called `graphical-session-good.service`, which is also ordered before - `boot-complete.target`, that registers a D-Bus endpoint. - -10. `systemd-boot-check-no-failures.service` is run after all its own - dependencies completed, and assesses that the boot completed - successfully. It hence exits cleanly. - -11. `graphical-session-good.service` waits for a user to log in. In the user - desktop environment, one minute after the user has logged in and started the - first program, a user service is invoked which makes a D-Bus call to - `graphical-session-good.service`. Upon receiving that call, - `graphical-session-good.service` exits cleanly. - -12. This allows `boot-complete.target` to be reached. This signifies to the - system that this boot attempt shall be considered successful. - -13. Which in turn permits `systemd-bless-boot.service` to run. It now - determines which boot loader entry file was used to boot the system, and - renames it dropping the counter tag. Thus - `4.14.11-300.fc27.x86_64+1-2.conf` is renamed to - `4.14.11-300.fc27.x86_64.conf`. From this moment boot counting is turned - off for this entry. - -14. On the following boot (and all subsequent boots after that) the entry is - now seen with boot counting turned off, no further renaming takes place. - -## How to adapt this scheme to other setups - -Of the stack described above many components may be replaced or augmented. Here -are a couple of recommendations. - -1. To support alternative boot loaders in place of `systemd-boot` two scenarios - are recommended: - - a. Boot loaders already implementing the Boot Loader Specification can - simply implement the same rename logic, and thus integrate fully with - the rest of the stack. - - b. Boot loaders that want to implement boot counting and store the counters - elsewhere can provide their own replacements for - `systemd-bless-boot.service` and `systemd-bless-boot-generator`, but should - continue to use `boot-complete.target` and thus support any services - ordered before that. - -2. To support additional components that shall succeed before the boot is - considered successful, simply place them in units (if they aren't already) - and order them before the generic `boot-complete.target` target unit, - combined with `Requires=` dependencies from the target, so that the target - cannot be reached when any of the units fail. You may add any number of - units like this, and only if they all succeed the boot entry is marked as - good. Note that the target unit shall pull in these boot checking units, not - the other way around. - - Depending on the setup, it may be most convenient to pull in such units - through normal enablement symlinks, or during early boot using a - [`generator`](https://www.freedesktop.org/software/systemd/man/systemd.generator.html), - or even during later boot. In the last case, care must be taken to ensure - that the start job is created before `boot-complete.target` has been - reached. - -3. To support additional components that shall only run on boot success, simply - wrap them in a unit and order them after `boot-complete.target`, pulling it - in. - - Such unit would be typically wanted (or required) by one of the - [`bootup`](https://www.freedesktop.org/software/systemd/man/bootup.html) targets, - for example, `multi-user.target`. To avoid potential loops due to conflicting - [default dependencies](https://www.freedesktop.org/software/systemd/man/systemd.unit.html#Default%20Dependencies) - ordering, it is recommended to also add an explicit dependency (e.g. - `After=multi-user.target`) to the unit. This overrides the implicit ordering - and allows `boot-complete.target` to start after the given bootup target. - -## FAQ - -1. *I have a service which — when it fails — should immediately cause a - reboot. How does that fit in with the above?* — That's orthogonal to - the above, please use `FailureAction=` in the unit file for this. - -2. *Under some condition I want to mark the current boot loader entry as bad - right-away, so that it never is tried again, how do I do that?* — You may - invoke `/usr/lib/systemd/systemd-bless-boot bad` at any time to mark the - current boot loader entry as "bad" right-away so that it isn't tried again - on later boots. diff --git a/docs/BACKPORTS.md b/docs/BACKPORTS.md deleted file mode 100644 index b2aba23a992..00000000000 --- a/docs/BACKPORTS.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Backports -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Backports - -The upstream systemd git repo at [https://github.com/systemd/systemd](https://github.com/systemd/systemd) only contains the main systemd branch that progresses at a quick pace, continuously bringing both bugfixes and new features. - -Distributions usually prefer basing their releases on stabilized versions branched off from this, that receive the bugfixes but not the features. - -## Stable Branch Repository - -Stable branches are available from [https://github.com/systemd/systemd-stable](https://github.com/systemd/systemd-stable). - -Stable branches are started for certain releases of systemd and named after them, e.g. v208-stable. -Stable branches are typically managed by distribution maintainers on an as needed basis. - -For example v208 has been chosen for stable as several distributions are shipping this version and the official/upstream cycle of v208-v209 was a long one due to kdbus work. - -If you are using a particular version and find yourself backporting several patches, you may consider pushing a stable branch here for that version so others can benefit. - -Please contact us if you are interested. - -The following types of commits are cherry-picked onto those branches: - -* bugfixes -* documentation updates, when relevant to this version -* hardware database additions, especially the keymap updates -* small non-conflicting features deemed safe to add in a stable release - -Please try to ensure that anything backported to the stable repository is done with the `git cherry-pick -x` option such that text stating the original SHA1 is added into the commit message. -This makes it easier to check where the code came from (as sometimes it is necessary to add small fixes as new code due to the upstream refactors) that are deemed too invasive to backport as a stable patch. diff --git a/docs/BOOT.md b/docs/BOOT.md deleted file mode 100644 index c72b31456b3..00000000000 --- a/docs/BOOT.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -title: systemd-boot UEFI Boot Manager -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# systemd-boot UEFI Boot Manager - -systemd-boot is a UEFI boot manager which executes configured EFI images. The default entry is selected by a configured pattern (glob) or an on-screen menu. - -systemd-boot operates on the EFI System Partition (ESP) only. Configuration file fragments, kernels, initrds, other EFI images need to reside on the ESP. - -Linux kernels need to be built with CONFIG\_EFI\_STUB to be able to be directly executed as an EFI image. - -systemd-boot reads simple and entirely generic boot loader configuration files; one file per boot loader entry to select from. All files need to reside on the ESP. - -Pressing the Space key (or most other keys actually work too) during bootup will show an on-screen menu with all configured loader entries to select from. - -Pressing Enter on the selected entry loads and starts the EFI image. - -If no timeout is configured, which is the default setting, and no key pressed during bootup, the default entry is executed right away. - -![systemd-boot menu](/assets/systemd-boot-menu.png) - -All configuration files are expected to be 7-bit ASCII or valid UTF8. The loader configuration file understands the following keywords: - -| Config | -|---------|------------------------------------------------------------| -| default | pattern to select the default entry in the list of entries | -| timeout | timeout in seconds for how long to show the menu | - - -The entry configuration files understand the following keywords: - -| Entry | -|--------|------------------------------------------------------------| -| title | text to show in the menu | -| version | version string to append to the title when the title is not unique | -| machine-id | machine identifier to append to the title when the title is not unique | -| efi | executable EFI image | -| options | options to pass to the EFI image / kernel command line | -| linux | linux kernel image (systemd-boot still requires the kernel to have an EFI stub) | -| initrd | initramfs image (systemd-boot just adds this as option initrd=) | - - -Examples: -``` -/boot/loader/loader.conf -timeout 3 -default 6a9857a393724b7a981ebb5b8495b9ea-* - -/boot/loader/entries/6a9857a393724b7a981ebb5b8495b9ea-3.8.0-2.fc19.x86_64.conf -title Fedora 19 (Rawhide) -version 3.8.0-2.fc19.x86_64 -machine-id 6a9857a393724b7a981ebb5b8495b9ea -linux /6a9857a393724b7a981ebb5b8495b9ea/3.8.0-2.fc19.x86_64/linux -initrd /6a9857a393724b7a981ebb5b8495b9ea/3.8.0-2.fc19.x86_64/initrd -options root=UUID=f8f83f73-df71-445c-87f7-31f70263b83b quiet - -/boot/loader/entries/custom-kernel.conf -title My kernel -efi /bzImage -options root=PARTUUID=084917b7-8be2-4e86-838d-f771a9902e08 - -/boot/loader/entries/custom-kernel-initrd.conf -title My kernel with initrd -linux /bzImage -initrd /initrd.img -options root=PARTUUID=084917b7-8be2-4e86-838d-f771a9902e08 quiet` -``` - - -While the menu is shown, the following keys are active: - -| Keys | -|--------|------------------------------------------------------------| -| Up/Down | Select menu entry | -| Enter | boot the selected entry | -| d | select the default entry to boot (stored in a non-volatile EFI variable) | -| t/T | adjust the timeout (stored in a non-volatile EFI variable) | -| e | edit the option line (kernel command line) for this bootup to pass to the EFI image | -| Q | quit | -| v | show the systemd-boot and UEFI version | -| P | print the current configuration to the console | -| h | show key mapping | - -Hotkeys to select a specific entry in the menu, or when pressed during bootup to boot the entry right-away: - - - -| Keys | -|--------|------------------------------------------------------------| -| l | Linux | -| w | Windows | -| a | macOS | -| s | EFI Shell | -| 1-9 | number of entry | - -Some EFI variables control the loader or exported the loaders state to the started operating system. The vendor UUID `4a67b082-0a4c-41cf-b6c7-440b29bb8c4f` and the variable names are supposed to be shared across all loaders implementations which follow this scheme of configuration: - -| EFI Variables | -|---------------|------------------------|-------------------------------| -| LoaderEntryDefault | entry identifier to select as default at bootup | non-volatile | -| LoaderEntrySysFail | sysfail entry identifier | non-volatile | -| LoaderSysFailReason | system failure reason | volatile | -| LoaderConfigTimeout | timeout in seconds to show the menu | non-volatile | -| LoaderEntryOneShot | entry identifier to select at the next and only the next bootup | non-volatile | -| LoaderDeviceIdentifier | list of identifiers of the volume the loader was started from | volatile | -| LoaderDevicePartUUID | partition GPT UUID of the ESP systemd-boot was executed from | volatile | - - -Links: - -[https://github.com/systemd/systemd](https://github.com/systemd/systemd) - -[https://uapi-group.org/specifications/specs/boot_loader_specification/](https://uapi-group.org/specifications/specs/boot_loader_specification/) diff --git a/docs/BOOT_LOADER_INTERFACE.md b/docs/BOOT_LOADER_INTERFACE.md deleted file mode 100644 index 981dd1f5516..00000000000 --- a/docs/BOOT_LOADER_INTERFACE.md +++ /dev/null @@ -1,174 +0,0 @@ ---- -title: Boot Loader Interface -category: Booting -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# The Boot Loader Interface - -systemd can interface with the boot loader to receive performance data and -other information, and pass control information. This is only supported on EFI -systems. Data is transferred between the boot loader and systemd in EFI -variables. All EFI variables use the vendor UUID -`4a67b082-0a4c-41cf-b6c7-440b29bb8c4f`. - -* The EFI Variable `LoaderTimeInitUSec` contains the timestamp in microseconds - when the loader was initialized. This value is the time spent in the firmware - for initialization, it is formatted as numeric, NUL-terminated, decimal - string, in UTF-16. - -* The EFI Variable `LoaderTimeExecUSec` contains the timestamp in microseconds - when the loader finished its work and is about to execute the kernel. The - time spent in the loader is the difference between `LoaderTimeExecUSec` and - `LoaderTimeInitUSec`. This value is formatted the same way as - `LoaderTimeInitUSec`. - -* The EFI variable `LoaderDevicePartUUID` contains the partition GUID of the - ESP the boot loader was run from formatted as NUL-terminated UTF16 string, in - normal GUID syntax. - -* The EFI variable `LoaderConfigTimeout` contains the boot menu timeout - currently in use. It may be modified both by the boot loader and by the - host. The value should be formatted as numeric, NUL-terminated, decimal - string, in UTF-16. The time is specified in seconds. In addition some - non-numeric string values are also accepted. A value of `menu-force` - will disable the timeout and show the menu indefinitely. If set to `0` or - `menu-hidden` the default entry is booted immediately without showing a menu. - Unless a value of `menu-disabled` is set, the boot loader should provide a - way to interrupt this by for example listening for key presses for a brief - moment before booting. - -* Similarly, the EFI variable `LoaderConfigTimeoutOneShot` contains a boot menu - timeout for a single following boot. It is set by the OS in order to request - display of the boot menu on the following boot. When set overrides - `LoaderConfigTimeout`. It is removed automatically after being read by the - boot loader, to ensure it only takes effect a single time. This value is - formatted the same way as `LoaderConfigTimeout`. If set to `0` the boot menu - timeout is turned off, and the menu is shown indefinitely. - -* The EFI variable `LoaderEntries` may contain a series of boot loader entry - identifiers, one after the other, each individually NUL terminated. This may - be used to let the OS know which boot menu entries were discovered by the - boot loader. A boot loader entry identifier should be a short, non-empty - alphanumeric string (possibly containing `-`, too). The list should be in the - order the entries are shown on screen during boot. See below regarding the - recommended vocabulary for boot loader entry identifiers. - -* The EFI variable `LoaderEntryDefault` contains the default boot loader entry - to use. It contains a NUL-terminated boot loader entry identifier. - -* The EFI variable `LoaderEntrySysFail` specifies the boot loader entry to be - used in case of a system failure. System failure (SysFail) boot entries can - optionally modify the automatic selection order in the event of a failure, - such as a boot firmware update failure with the failure status recorded in - the EFI system table. If a system failure occurs and `LoaderEntrySysFail` is - set, systemd-boot will use this boot entry, and store the actual SysFail - reason in the `LoaderSysFailReason` EFI variable. - -* The EFI variable `LoaderSysFailReason` contains the system failure reason. - This variable is used in cooperation with `LoaderEntrySysFail` boot entry. - If system failure doesn't occur, `LoaderSysFailReason` is not set. - -* Similarly, the EFI variable `LoaderEntryOneShot` contains the default boot - loader entry to use for a single following boot. It is set by the OS in order - to request booting into a specific menu entry on the following boot. When set - overrides `LoaderEntryDefault`. It is removed automatically after being read - by the boot loader, to ensure it only takes effect a single time. This value - is formatted the same way as `LoaderEntryDefault`. - -* The EFI variable `LoaderEntrySelected` contains the boot loader entry - identifier that was booted. It is set by the boot loader and read by - the OS in order to identify which entry has been used for the current boot. - -* The EFI variable `LoaderFeatures` contains a 64-bit unsigned integer with a - number of flags bits that are set by the boot loader and passed to the OS and - indicate the features the boot loader supports. Specifically, the following - bits are defined: - - * `1 << 0` → The boot loader honours `LoaderConfigTimeout` when set. - * `1 << 1` → The boot loader honours `LoaderConfigTimeoutOneShot` when set. - * `1 << 2` → The boot loader honours `LoaderEntryDefault` when set. - * `1 << 3` → The boot loader honours `LoaderEntryOneShot` when set. - * `1 << 4` → The boot loader supports boot counting as described in [Automatic Boot Assessment](/AUTOMATIC_BOOT_ASSESSMENT). - * `1 << 5` → The boot loader supports looking for boot menu entries in the Extended Boot Loader Partition. - * `1 << 6` → The boot loader supports passing a random seed to the OS. - * `1 << 13` → The boot loader honours `menu-disabled` option when set. - -* The EFI variable `LoaderSystemToken` contains binary random data, - persistently set by the OS installer. Boot loaders that support passing - random seeds to the OS should use this data and combine it with the random - seed file read from the ESP. By combining this random data with the random - seed read off the disk before generating a seed to pass to the OS and a new - seed to store in the ESP the boot loader can protect itself from situations - where "golden" OS images that include a random seed are replicated and used - on multiple systems. Since the EFI variable storage is usually independent - (i.e. in physical NVRAM) of the ESP file system storage, and only the latter - is part of "golden" OS images, this ensures that different systems still come - up with different random seeds. Note that the `LoaderSystemToken` is - generally only written once, by the OS installer, and is usually not touched - after that. - -* The EFI variable `LoaderDeviceURL` contains the URL the boot loader was - downloaded from, in UTF-16 format. Only set in case of network boots. - -* The EFI variable `LoaderTpm2ActivePcrBanks` contains a hexadecimal string - representation of a bitmask with values defined by the TCG EFI Protocol - Specification for TPM 2.0 as EFI_TCG2_BOOT_HASH_ALG_*. If no TPM2 support or - no active banks were detected, will be set to `0`. - -If `LoaderTimeInitUSec` and `LoaderTimeExecUSec` are set, `systemd-analyze` -will include them in its boot-time analysis. If `LoaderDevicePartUUID` is set, -systemd will mount the ESP that was used for the boot to `/boot`, but only if -that directory is empty, and only if no other file systems are mounted -there. The `systemctl reboot --boot-loader-entry=…` and `systemctl reboot ---boot-loader-menu=…` commands rely on the `LoaderFeatures` , -`LoaderConfigTimeoutOneShot`, `LoaderEntries`, `LoaderEntryOneShot` -variables. - -## Boot Loader Entry Identifiers - -While boot loader entries may be named relatively freely, it's highly -recommended to follow the following rules when picking identifiers for the -entries, so that programs (and users) can derive basic context and meaning from -the identifiers as passed in `LoaderEntries`, `LoaderEntryDefault`, -`LoaderEntryOneShot`, `LoaderEntrySelected`, and possibly show nicely localized -names for them in UIs. - -1. When boot loader entries are defined through the - [Boot Loader Specification](https://uapi-group.org/specifications/specs/boot_loader_specification/) - files, the identifier should be derived directly from the file name, - but with the `.conf` (Type #1 snippets) or `.efi` (Type #2 images) - suffix removed. - -2. Entries automatically discovered by the boot loader (as opposed to being - configured in configuration files) should generally have an identifier - prefixed with `auto-`. - -3. Boot menu entries referring to Microsoft Windows installations should either - use the identifier `windows` or use the `windows-` prefix for the - identifier. If a menu entry is automatically discovered, it should be - prefixed with `auto-`, see above (Example: this means an automatically - discovered Windows installation might have the identifier `auto-windows` or - `auto-windows-10` or so.). - -4. Similarly, boot menu entries referring to Apple macOS installations should - use the identifier `osx` or one that is prefixed with `osx-`. If such an - entry is automatically discovered by the boot loader use `auto-osx` as - identifier, or `auto-osx-` as prefix for the identifier, see above. - -5. If a boot menu entry encapsulates the EFI shell program, it should use the - identifier `efi-shell` (or when automatically discovered: `auto-efi-shell`, - see above). - -6. If a boot menu entry encapsulates a reboot into EFI firmware setup feature, - it should use the identifier `reboot-to-firmware-setup` (or - `auto-reboot-to-firmware-setup` in case it is automatically discovered). - -## Links - -[Boot Loader Specification](https://uapi-group.org/specifications/specs/boot_loader_specification)
-[Discoverable Partitions Specification](https://uapi-group.org/specifications/specs/discoverable_partitions_specification)
-[`systemd-boot(7)`](https://www.freedesktop.org/software/systemd/man/systemd-boot.html)
-[`bootctl(1)`](https://www.freedesktop.org/software/systemd/man/bootctl.html)
-[`systemd-gpt-auto-generator(8)`](https://www.freedesktop.org/software/systemd/man/systemd-gpt-auto-generator.html) diff --git a/docs/BOOT_LOADER_SPECIFICATION.md b/docs/BOOT_LOADER_SPECIFICATION.md deleted file mode 100644 index 33066b2875a..00000000000 --- a/docs/BOOT_LOADER_SPECIFICATION.md +++ /dev/null @@ -1 +0,0 @@ -[This content has moved to the UAPI group website](https://uapi-group.org/specifications/specs/boot_loader_specification/) diff --git a/docs/CGROUP_DELEGATION.md b/docs/CGROUP_DELEGATION.md deleted file mode 100644 index 2151bca9217..00000000000 --- a/docs/CGROUP_DELEGATION.md +++ /dev/null @@ -1,427 +0,0 @@ ---- -title: Control Group APIs and Delegation -category: Interfaces -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Control Group APIs and Delegation - -*Intended audience: hackers working on userspace subsystems that require direct -cgroup access, such as container managers and similar.* - -So you are wondering about resource management with systemd, you know Linux -control groups (cgroups) a bit and are trying to integrate your software with -what systemd has to offer there. Here's a bit of documentation about the -concepts and interfaces involved with this. - -What's described here has been part of systemd and documented since v205 -times. However, it has been updated and improved substantially, even -though the concepts stayed mostly the same. This is an attempt to provide more -comprehensive up-to-date information about all this, particular in light of the -poor implementations of the components interfacing with systemd of current -container managers. - -Before you read on, please make sure you read the low-level kernel -documentation about the -[unified cgroup hierarchy](https://docs.kernel.org/admin-guide/cgroup-v2.html). -This document then adds in the higher-level view from systemd. - -This document augments the existing documentation we already have: - -* [The New Control Group Interfaces](/CONTROL_GROUP_INTERFACE) -* [Writing VM and Container Managers](/WRITING_VM_AND_CONTAINER_MANAGERS) - -These wiki documents are not as up to date as they should be, currently, but -the basic concepts still fully apply. You should read them too, if you do something -with cgroups and systemd, in particular as they shine more light on the various -D-Bus APIs provided. (That said, sooner or later we should probably fold that -wiki documentation into this very document, too.) - -## Two Key Design Rules - -Much of the philosophy behind these concepts is based on a couple of basic -design ideas of cgroup v2. Specifically two cgroup v2 rules are the most relevant: - -1. The **no-processes-in-inner-nodes** rule: this means that it's not permitted -to have processes directly attached to a cgroup that also has child cgroups and -vice versa. A cgroup is either an inner node or a leaf node of the tree, and if -it's an inner node it may not contain processes directly, and if it's a leaf -node then it may not have child cgroups. (Note that there are some minor -exceptions to this rule, though. E.g. the root cgroup is special and allows -both processes and children — which is used in particular to maintain kernel -threads.) - -2. The **single-writer** rule: this means that each cgroup only has a single -writer, i.e. a single process managing it. It's OK if different cgroups have -different processes managing them. However, only a single process should own a -specific cgroup, and when it does that ownership is exclusive, and nothing else -should manipulate it at the same time. This rule ensures that various pieces of -software don't step on each other's toes constantly. - -These two rules have various effects. For example, one corollary of this is: if -your container manager creates and manages cgroups in the system's root cgroup -you violate rule #2, as the root cgroup is managed by systemd and hence off -limits to everybody else. - -Note that rule #1 is generally enforced by the kernel if cgroup v2 is used: as -soon as you add a process to a cgroup it is ensured the rule is not -violated. On cgroup v1 this rule didn't exist, and hence isn't enforced, even -though it's a good thing to follow it then too. Rule #2 is not enforced on -either cgroup v1 nor cgroup v2 (this is UNIX after all, in the general case -root can do anything, modulo SELinux and friends), but if you ignore it you'll -be in constant pain as various pieces of software will fight over cgroup -ownership. - -Note that cgroup v1 is semantically broken in many ways, and in many cases doesn't -actually do what people think it does. cgroup v2 is where things are going, and -new kernel features in this area are only added to cgroup v2, and not cgroup v1 -anymore. For example, cgroup v2 provides proper cgroup-empty notifications, has -support for all kinds of per-cgroup BPF magic, supports secure delegation of -cgroup trees to less privileged processes and so on, which all are not -available on cgroup v1. - -## Hierarchy and Controller Support - -systemd supports what's called **unified** hierarchy, which is a simply mode -that exposes a pure cgroup v2 logic. In this mode `/sys/fs/cgroup/` is the only -mounted cgroup API file system and all available controllers are exclusively -exposed through it. - -systemd supports a number of controllers (but not all). Specifically, supported -are: `cpu`, `io`, `memory`, `pids`. It is our intention to natively support all -cgroup v2 controllers as they are added to the kernel. - -The following hierarchies were deprecated in v256, and eventually got removed -in v258: - -* **Legacy** — this is the traditional cgroup v1 mode. In this mode the -various controllers each get their own cgroup file system mounted to -`/sys/fs/cgroup//`. On top of that systemd manages its own cgroup -hierarchy for managing purposes as `/sys/fs/cgroup/systemd/`. - -* **Hybrid** — this is a hybrid between the unified and legacy mode. It's set -up mostly like legacy, except that there's also an additional hierarchy -`/sys/fs/cgroup/unified/` that contains the cgroup v2 hierarchy. (Note that in -this mode the unified hierarchy won't have controllers attached, the -controllers are all mounted as separate hierarchies as in legacy mode, -i.e. `/sys/fs/cgroup/unified/` is purely and exclusively about core cgroup v2 -functionality and not about resource management.) In this mode compatibility -with cgroup v1 is retained while some cgroup v2 features are available -too. This mode is a stopgap. Don't bother with this too much unless you have -too much free time. - -## systemd's Unit Types - -The low-level kernel cgroups feature is exposed in systemd in three different -"unit" types. Specifically: - -1. 💼 The `.service` unit type. This unit type is for units encapsulating - processes systemd itself starts. Units of these types have cgroups that are - the leaves of the cgroup tree the systemd instance manages (though possibly - they might contain a sub-tree of their own managed by something else, made - possible by the concept of delegation, see below). Service units are usually - instantiated based on a unit file on disk that describes the command line to - invoke and other properties of the service. However, service units may also - be declared and started programmatically at runtime through a D-Bus API - (which is called *transient* services). - -2. 👓 The `.scope` unit type. This is very similar to `.service`. The main - difference: the processes the units of this type encapsulate are forked off - by some unrelated manager process, and that manager asked systemd to expose - them as a unit. Unlike services, scopes can only be declared and started - programmatically, i.e. are always transient. That's because they encapsulate - processes forked off by something else, i.e. existing runtime objects, and - hence cannot really be defined fully in 'offline' concepts such as unit - files. - -3. 🔪 The `.slice` unit type. Units of this type do not directly contain any - processes. Units of this type are the inner nodes of part of the cgroup tree - the systemd instance manages. Much like services, slices can be defined - either on disk with unit files or programmatically as transient units. - -Slices expose the trunk and branches of a tree, and scopes and services are -attached to those branches as leaves. The idea is that scopes and services can -be moved around though, i.e. assigned to a different slice if needed. - -The naming of slice units directly maps to the cgroup tree path. This is not -the case for service and scope units however. A slice named `foo-bar-baz.slice` -maps to a cgroup `/foo.slice/foo-bar.slice/foo-bar-baz.slice/`. A service -`quux.service` which is attached to the slice `foo-bar-baz.slice` maps to the -cgroup `/foo.slice/foo-bar.slice/foo-bar-baz.slice/quux.service/`. - -By default systemd sets up four slice units: - -1. `-.slice` is the root slice. i.e. the parent of everything else. On the host - system it maps directly to the top-level directory of cgroup v2. - -2. `system.slice` is where system services are by default placed, unless - configured otherwise. - -3. `user.slice` is where user sessions are placed. Each user gets a slice of - its own below that. - -4. `machines.slice` is where VMs and containers are supposed to be - placed. `systemd-nspawn` makes use of this by default, and you're very welcome - to place your containers and VMs there too if you hack on managers for those. - -Users may define any amount of additional slices they like though, the four -above are just the defaults. - -## Delegation - -Container managers and suchlike often want to control cgroups directly using -the raw kernel APIs. That's entirely fine and supported, as long as proper -*delegation* is followed. Delegation is a concept we inherited from cgroup v2. -It means that some parts of the cgroup tree may be managed by different managers -than others. As long as it is clear which manager manages which part of the tree -each one can do within its sub-graph of the tree whatever it wants. - -Only sub-trees can be delegated (though whoever decides to request a sub-tree -can delegate sub-sub-trees further to somebody else if they like). Delegation -takes place at a specific cgroup: in systemd there's a `Delegate=` property you -can set for a service or scope unit. If you do, it's the cut-off point for -systemd's cgroup management: the unit itself is managed by systemd, i.e. all -its attributes are managed exclusively by systemd, however your program may -create/remove sub-cgroups inside it freely, and those then become exclusive -property of your program, systemd won't touch them — all attributes of *those* -sub-cgroups can be manipulated freely and exclusively by your program. - -By turning on the `Delegate=` property for a scope or service you get a few -guarantees: - -1. systemd won't fiddle with your sub-tree of the cgroup tree anymore. It won't - change attributes of any cgroups below it, nor will it create or remove any - cgroups thereunder, nor migrate processes across the boundaries of that - sub-tree as it deems useful anymore. - -2. If your service makes use of the `User=` functionality, then the sub-tree - will be `chown()`ed to the indicated user so that it can correctly create - cgroups below it. - -3. Any BPF IP filter programs systemd installs will be installed with - `BPF_F_ALLOW_MULTI` so that your program can install additional ones. - -In unit files the `Delegate=` property is superficially exposed as -boolean. However, since v236 it optionally takes a list of controller names -instead. If so, delegation is requested for listed controllers -specifically. Note that this only encodes a request. Depending on various -parameters it might happen that your service actually will get fewer -controllers delegated (for example, because the controller is not available on -the current kernel or was turned off) or more. If no list is specified -(i.e. the property simply set to `yes`) then all available controllers are -delegated. - -Let's stress one thing: delegation is available on scope and service units -only. It's expressly not available on slice units. Why? Because slice units are -our *inner* nodes of the cgroup trees and we freely attach services and scopes -to them. If we'd allow delegation on slice units then this would mean that -both systemd and your own manager would create/delete cgroups below the slice -unit and that conflicts with the single-writer rule. - -So, if you want to do your own raw cgroups kernel level access, then allocate a -scope unit, or a service unit (or just use the service unit you already have -for your service code), and turn on delegation for it. - -The service manager sets the `user.delegate` extended attribute (readable via -`getxattr(2)` and related calls) to the character `1` on cgroup directories -where delegation is enabled (and removes it on those cgroups where it is -not). This may be used by service programs to determine whether a cgroup tree -was delegated to them. Note that this is only supported on kernels 5.6 and -newer in combination with systemd 251 and newer. - -(OK, here's one caveat: if you turn on delegation for a service, and that -service has `ExecStartPost=`, `ExecReload=`, `ExecStop=` or `ExecStopPost=` -set, then these commands will be executed within the `.control/` sub-cgroup of -your service's cgroup. This is necessary because by turning on delegation we -have to assume that the cgroup delegated to your service is now an *inner* -cgroup, which means that it may not directly contain any processes. Hence, if -your service has any of these four settings set, you must be prepared that a -`.control/` subcgroup might appear, managed by the service manager. This also -means that your service code should have moved itself further down the cgroup -tree by the time it notifies the service manager about start-up readiness, so -that the service's main cgroup is definitely an inner node by the time the -service manager might start `ExecStartPost=`. Starting with systemd 254 you may -also use `DelegateSubgroup=` to let the service manager put your initial -service process into a subgroup right away.) - -(Also note, if you intend to use "threaded" cgroups — as added in Linux 4.14 —, -then you should do that *two* levels down from the main service cgroup your -turned delegation on for. Why that? You need one level so that systemd can -properly create the `.control` subgroup, as described above. But that one -cannot be threaded, since that would mean `.control` has to be threaded too — -this is a requirement of threaded cgroups: either a cgroup and all its siblings -are threaded or none –, but systemd expects it to be a regular cgroup. Thus you -have to nest a second cgroup beneath it which then can be threaded.) - -Finally, if you turn on cgroup delegation with `Delegate=yes`, systemd will make -the requested controllers available to your service or scope, but won't actually -enable them. Currently you have to do that manually by writing to -`cgroup.subtree_control` within your delegated cgroup (e.g. write `+memory` to -enable the memory controller). - -## Three Scenarios - -Let's say you write a container manager, and you wonder what to do regarding -cgroups for it, as you want your manager to be able to run on systemd systems. - -You basically have three options: - -1. 😊 The *integration-is-good* option. For this, you register each container - you have either as a systemd service (i.e. let systemd invoke the executor - binary for you) or a systemd scope (i.e. your manager executes the binary - directly, but then tells systemd about it. In this mode the administrator - can use the usual systemd resource management and reporting commands - individually on those containers. By turning on `Delegate=` for these scopes - or services you make it possible to run cgroup-enabled programs in your - containers, for example a nested systemd instance. This option has two - sub-options: - - a. You transiently register the service or scope by directly contacting - systemd via D-Bus. In this case systemd will just manage the unit for you - and nothing else. - - b. Instead you register the service or scope through `systemd-machined` - (also via D-Bus). This mini-daemon is basically just a proxy for the same - operations as in a. The main benefit of this: this way you let the system - know that what you are registering is a container, and this opens up - certain additional integration points. For example, `journalctl -M` can - then be used to directly look into any container's journal logs (should - the container run systemd inside), or `systemctl -M` can be used to - directly invoke systemd operations inside the containers. Moreover tools - like "ps" can then show you to which container a process belongs (`ps -eo - pid,comm,machine`), and even gnome-system-monitor supports it. - -2. 🙁 The *i-like-islands* option. If all you care about is your own cgroup tree, - and you want to have to do as little as possible with systemd and no - interest in integration with the rest of the system, then this is a valid - option. For this all you have to do is turn on `Delegate=` for your main - manager daemon. Then figure out the cgroup systemd placed your daemon in: - you can now freely create sub-cgroups beneath it. Don't forget the - *no-processes-in-inner-nodes* rule however: you have to move your main - daemon process out of that cgroup (and into a sub-cgroup) before you can - start further processes in any of your sub-cgroups. - -3. 🙁 The *i-like-continents* option. In this option you'd leave your manager - daemon where it is, and would not turn on delegation on its unit. However, - as you start your first managed process (a container, for example) you would - register a new scope unit with systemd, and that scope unit would have - `Delegate=` turned on, and it would contain the PID of this process; all - your managed processes subsequently created should also be moved into this - scope. From systemd's PoV there'd be two units: your manager service and the - big scope that contains all your managed processes in one. - -BTW: if for whatever reason you say "I hate D-Bus, I'll never call any D-Bus -API, kthxbye", then options #1 and #3 are not available, as they generally -involve talking to systemd from your program code, via D-Bus. You still have -option #2 in that case however, as you can simply set `Delegate=` in your -service's unit file and you are done and have your own sub-tree. In fact, #2 is -the one option that allows you to completely ignore systemd's existence: you -can entirely generically follow the single rule that you just use the cgroup -you are started in, and everything below it, whatever that might be. That said, -maybe if you dislike D-Bus and systemd that much, the better approach might be -to work on that, and widen your horizon a bit. You are welcome. - -## systemd as Container Payload - -systemd can happily run as a container payload's PID 1. Note that systemd -unconditionally needs write access to the cgroup tree however, hence you need -to delegate a sub-tree to it. Note that there's nothing too special you have to -do beyond that: just invoke systemd as PID 1 inside the root of the delegated -cgroup sub-tree, and it will figure out the rest: it will determine the cgroup -it is running in and take possession of it. It won't interfere with any cgroup -outside of the sub-tree it was invoked in. Use of `CLONE_NEWCGROUP` is hence -optional (but of course wise). - -Note one particular asymmetry here though: systemd will try to take possession -of the root cgroup you pass to it *in* *full*, i.e. it will not only -create/remove child cgroups below it, it will also attempt to manage the -attributes of it. OTOH as mentioned above, when delegating a cgroup tree to -somebody else it only passes the rights to create/remove sub-cgroups, but will -insist on managing the delegated cgroup tree's top-level attributes. Or in -other words: systemd is *greedy* when accepting delegated cgroup trees and also -*greedy* when delegating them to others: it insists on managing attributes on -the specific cgroup in both cases. A container manager that is itself a payload -of a host systemd which wants to run a systemd as its own container payload -instead hence needs to insert an extra level in the hierarchy in between, so -that the systemd on the host and the one in the container won't fight for the -attributes. That said, you likely should do that anyway, due to the -no-processes-in-inner-cgroups rule, see below. - -When systemd runs as container payload it will make use of all hierarchies it -has write access to. For legacy mode you need to make at least -`/sys/fs/cgroup/systemd/` available, all other hierarchies are optional. For -hybrid mode you need to add `/sys/fs/cgroup/unified/`. Finally, for fully -unified you (of course, I guess) need to provide only `/sys/fs/cgroup/` itself. - -## Some Dos - -1. ⚡ If you go for implementation option 1a or 1b (as in the list above), then - each of your containers will have its own systemd-managed unit and hence - cgroup with possibly further sub-cgroups below. Typically the first process - running in that unit will be some kind of executor program, which will in - turn fork off the payload processes of the container. In this case don't - forget that there are two levels of delegation involved: first, systemd - delegates a group sub-tree to your executor. And then your executor should - delegate a sub-tree further down to the container payload. Oh, and because - of the no-process-in-inner-nodes rule, your executor needs to migrate itself - to a sub-cgroup of the cgroup it got delegated, too. Most likely you hence - want a two-pronged approach: below the cgroup you got started in, you want - one cgroup maybe called `supervisor/` where your manager runs in and then - for each container a sibling cgroup of that maybe called `payload-xyz/`. - -2. ⚡ Don't forget that the cgroups you create have to have names that are - suitable as UNIX file names, and that they live in the same namespace as the - various kernel attribute files. Hence, when you want to allow the user - arbitrary naming, you might need to escape some of the names (for example, - you really don't want to create a cgroup named `tasks`, just because the - user created a container by that name, because `tasks` after all is a magic - attribute in cgroup v1, and your `mkdir()` will hence fail with `EEXIST`. In - systemd we do escaping by prefixing names that might collide with a kernel - attribute name with an underscore. You might want to do the same, but this - is really up to you how you do it. Just do it, and be careful. - -## Some Don'ts - -1. 🚫 Never create your own cgroups below arbitrary cgroups systemd manages, i.e - cgroups you haven't set `Delegate=` in. Specifically: 🔥 don't create your - own cgroups below the root cgroup 🔥. That's owned by systemd, and you will - step on systemd's toes if you ignore that, and systemd will step on - yours. Get your own delegated sub-tree, you may create as many cgroups there - as you like. Seriously, if you create cgroups directly in the cgroup root, - then all you do is ask for trouble. - -2. 🚫 Don't attempt to set `Delegate=` in slice units, and in particular not in - `-.slice`. It's not supported, and will generate an error. - -3. 🚫 Never *write* to any of the attributes of a cgroup systemd created for - you. It's systemd's private property. You are welcome to manipulate the - attributes of cgroups you created in your own delegated sub-tree, but the - cgroup tree of systemd itself is out of limits for you. It's fine to *read* - from any attribute you like however. That's totally OK and welcome. - -4. 🚫 When not using `CLONE_NEWCGROUP` when delegating a sub-tree to a - container payload running systemd, then don't get the idea that you can bind - mount only a sub-tree of the host's cgroup tree into the container. Part of - the cgroup API is that `/proc/$PID/cgroup` reports the cgroup path of every - process, and hence any path below `/sys/fs/cgroup/` needs to match what - `/proc/$PID/cgroup` of the payload processes reports. What you can do safely - however, is mount the upper parts of the cgroup tree read-only (or even - replace the middle bits with an intermediary `tmpfs` — but be careful not to - break the `statfs()` detection logic discussed above), as long as the path - to the delegated sub-tree remains accessible as-is. - -5. ⚡ Currently, the algorithm for mapping between slice/scope/service unit - naming and their cgroup paths is not considered public API of systemd, and - may change in future versions. This means: it's best to avoid implementing a - local logic of translating cgroup paths to slice/scope/service names in your - program, or vice versa — it's likely going to break sooner or later. Use the - appropriate D-Bus API calls for that instead, so that systemd translates - this for you. (Specifically: each Unit object has a `ControlGroup` property - to get the cgroup for a unit. The method `GetUnitByControlGroup()` may be - used to get the unit for a cgroup.) - -And that's it for now. If you have further questions, refer to the systemd -mailing list. - -— Berlin, 2018-04-20 diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md index 08ccc30d798..540ec246bc6 100644 --- a/docs/CODE_OF_CONDUCT.md +++ b/docs/CODE_OF_CONDUCT.md @@ -1,14 +1,14 @@ --- -title: systemd Community Conduct Guidelines +title: FileSystemds Community Conduct Guidelines category: Contributing layout: default SPDX-License-Identifier: LGPL-2.1-or-later --- -# The systemd Community Conduct Guidelines +# FileSystemds Community Conduct Guidelines -This document provides community guidelines for a safe, respectful, productive, and collaborative place for any person who is willing to contribute to systemd. -It applies to all “collaborative spaces”, which is defined as community communications channels (such as mailing lists, submitted patches, commit comments, etc.). +This document provides community guidelines for a safe, respectful, productive, and collaborative place for any person who is willing to contribute to the FileSystemds Mobile Platform project. +It applies to all "collaborative spaces", which is defined as community communications channels (such as GitHub Issues, Pull Requests, commit comments, etc.). - Participants will be tolerant of opposing views. - Participants must ensure that their language and actions are free of personal attacks and disparaging personal remarks. @@ -17,9 +17,7 @@ It applies to all “collaborative spaces”, which is defined as community comm ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at systemd-conduct@googlegroups.com. -This team currently consists of David Strauss <>, Ekaterina Gerasimova (Kat) <>, and Zbigniew Jędrzejewski-Szmek <>. -In the unfortunate event that you wish to make a complaint against one of the members, you may instead contact any of the other members individually. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by creating a GitHub Issue in this repository or contacting the project maintainers directly through GitHub. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. -The project team is obligated to maintain confidentiality with regard to the reporter of an incident. +The project team is obligated to maintain confidentiality with regard to the reporter of an incident. \ No newline at end of file diff --git a/docs/CONTAINER_INTERFACE.md b/docs/CONTAINER_INTERFACE.md deleted file mode 100644 index 2a823218814..00000000000 --- a/docs/CONTAINER_INTERFACE.md +++ /dev/null @@ -1,425 +0,0 @@ ---- -title: Container Interface -category: Interfaces -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# The Container Interface - -Also consult [Writing Virtual Machine or Container Managers](/WRITING_VM_AND_CONTAINER_MANAGERS). - -systemd has a number of interfaces for interacting with container managers, -when systemd is used inside of an OS container. If you work on a container -manager, please consider supporting the following interfaces. - -## Execution Environment - -1. If the container manager wants to control the hostname for a container - running systemd it may just set it before invoking systemd, and systemd will - leave it unmodified when there is no hostname configured in `/etc/hostname` - (that file overrides whatever is pre-initialized by the container manager). - -2. Make sure to pre-mount `/proc/`, `/sys/`, and `/sys/fs/selinux/` before - invoking systemd, and mount `/sys/`, `/sys/fs/selinux/` and `/proc/sys/` - read-only (the latter via e.g. a read-only bind mount on itself) in order - to prevent the container from altering the host kernel's configuration - settings. As a special exception, if your container has network namespaces - enabled, feel free to make `/proc/sys/net/` writable. If it also has user, ipc, - uts and pid namespaces enabled, the entire `/proc/sys` can be left writable. - However, in the latter case, an appropriate userns mapping should exist to - map the root user inside the container to an unprivileged user on the - host. Otherwise, the root user inside the container could modify the host's - kernel settings. systemd and various other subsystems (such as the SELinux - userspace) have been modified to behave accordingly when these file systems - are read-only. (It's OK to mount `/sys/` as `tmpfs` btw, and only mount a - subset of its sub-trees from the real `sysfs` to hide `/sys/firmware/`, - `/sys/kernel/` and so on. If you do that, make sure to mark `/sys/` - read-only, as that condition is what systemd looks for, and is what is - considered to be the API in this context.) - -3. Pre-mount `/dev/` as (container private) `tmpfs` for the container and bind - mount some suitable TTY to `/dev/console`. If this is a pty, make sure to - not close the controlling pty during systemd's lifetime. PID 1 will close - ttys, to avoid being killed by SAK. It only opens ttys for the time it - actually needs to print something. Also, make sure to create device nodes - for `/dev/null`, `/dev/zero`, `/dev/full`, `/dev/random`, `/dev/urandom`, - `/dev/tty`, `/dev/ptmx` in `/dev/`. It is not necessary to create `/dev/fd` - or `/dev/stdout`, as systemd will do that on its own. Make sure to set up a - `BPF_PROG_TYPE_CGROUP_DEVICE` BPF program on cgroupv2, so that no other devices - but these may be created in the container. Note that many systemd services use - `PrivateDevices=`, which means that systemd will set up a private `/dev/` - for them for which it needs to be able to create these device nodes. - Dropping `CAP_MKNOD` for containers is hence generally not advisable, but - see below. - -4. `systemd-udevd` is not available in containers (and refuses to start), and - hence device dependencies are unavailable. The `systemd-udevd` unit files - will check for `/sys/` being read-only, as an indication whether device - management can work. Therefore make sure to mount `/sys/` read-only in the - container (see above). Various clients of `systemd-udevd` also check the - read-only state of `/sys/`, including PID 1 itself and `systemd-networkd`. - -5. If systemd detects it is run in a container it will spawn a single shell on - `/dev/console`, and not care about VTs or multiple gettys on VTs. (But see - `$container_ttys` below.) - -6. Either pre-mount all cgroup hierarchies in full into the container, or leave - that to systemd which will do so if they are missing. Note that it is - explicitly *not* OK to just mount a sub-hierarchy into the container as that - is incompatible with `/proc/$PID/cgroup` (which lists full paths). Also the - root-level cgroup directories tend to be quite different from inner - directories, and that distinction matters. It is OK however, to mount the - "upper" parts read-only of the hierarchies, and only allow write-access to - the cgroup sub-tree the container runs in. - -7. Create the control group root of your container by either running your - container as a service (in case you have one container manager instance per - container instance) or creating one scope unit for each container instance - via systemd's transient unit API (in case you have one container manager - that manages all instances. Either way, make sure to set `Delegate=yes` in - it. This ensures that the unit you created will be part of all cgroup - controllers (or at least the ones systemd understands). The latter may also - be done via `systemd-machined`'s `CreateMachine()` API. Make sure to use the - cgroup path systemd put your process in for all operations of the container. - Do not add new cgroup directories to the top of the tree. This will not only - confuse systemd and the admin, but also prevent your implementation from - being "stackable". - -## Environment Variables - -1. To allow systemd (and other programs) to identify that it is executed within - a container, please set the `$container` environment variable for PID 1 in - the container to a short lowercase string identifying your - implementation. With this in place the `ConditionVirtualization=` setting in - unit files will work properly. Example: `container=lxc-libvirt` - -2. systemd has special support for allowing container managers to initialize - the UUID for `/etc/machine-id` to some manager supplied value. This is only - enabled if `/etc/machine-id` is empty (i.e. not yet set) at boot time of the - container. The container manager should set `$container_uuid` as environment - variable for the container's PID 1 to the container UUID. (This is similar - to the effect of `qemu`'s `-uuid` switch). Note that you should pass only a - UUID here that is actually unique (i.e. only one running container should - have a specific UUID), and gets changed when a container gets duplicated. - Also note that systemd will try to persistently store the UUID in - `/etc/machine-id` (if writable) when this option is used, hence you should - always pass the same UUID here. Keeping the externally used UUID for a - container and the internal one in sync is hopefully useful to minimize - surprise for the administrator. - -3. systemd can automatically spawn login gettys on additional ptys. A container - manager can set the `$container_ttys` environment variable for the - container's PID 1 to tell it on which ptys to spawn gettys. The variable - should take a space separated list of pty names, without the leading `/dev/` - prefix, but with the `pts/` prefix included. Note that despite the - variable's name you may only specify ptys, and not other types of ttys. Also - you need to specify the pty itself, a symlink will not suffice. This is - implemented in - [systemd-getty-generator(8)](https://www.freedesktop.org/software/systemd/man/latest/systemd-getty-generator.html). - Note that this variable should not include the pty that `/dev/console` maps - to if it maps to one (see below). Example: if the container receives - `container_ttys=pts/7 pts/8 pts/14` it will spawn three additional login - gettys on ptys 7, 8, and 14. - -4. To allow applications to detect the OS version and other metadata of the host - running the container manager, if this is considered desirable, please parse - the host's `/etc/os-release` and set a `$container_host_=` - environment variable for the ID fields described by the [os-release - interface](https://www.freedesktop.org/software/systemd/man/latest/os-release.html), eg: - `$container_host_id=debian` - `$container_host_build_id=2020-06-15` - `$container_host_variant_id=server` - `$container_host_version_id=10` - -5. systemd supports passing immutable binary data blobs with limited size and - restricted access to services via the `ImportCredential=`, `LoadCredential=` - and `SetCredential=` settings. The same protocol may be used to pass credentials - from the container manager to systemd itself. The credential data should be - placed in some location (ideally a read-only and non-swappable file system, - like 'ramfs'), and the absolute path to this directory exported in the - `$CREDENTIALS_DIRECTORY` environment variable. If the container managers - does this, the credentials passed to the service manager can be propagated - to services via `LoadCredential=` or `ImportCredential=` (see ...). The - container manager can choose any path, but `/run/host/credentials` is - recommended. - -## Advanced Integration - -1. Consider syncing `/etc/localtime` from the host file system into the - container. Make it a relative symlink to the containers's zoneinfo dir, as - usual. Tools rely on being able to determine the timezone setting from the - symlink value, and making it relative looks nice even if people list the - container's `/etc/` from the host. - -2. Make the container journal available in the host, by automatically - symlinking the container journal directory into the host journal directory. - More precisely, link `/var/log/journal/` of the - container into the same dir of the host. Administrators can then - automatically browse all container journals (correctly interleaved) by - issuing `journalctl -m`. The container machine ID can be determined from - `/etc/machine-id` in the container. - -3. If the container manager wants to cleanly shut down the container, it might - be a good idea to send `SIGRTMIN+3` to its init process. systemd will then - do a clean shutdown. Note however, that since only systemd understands - `SIGRTMIN+3` like this, this might confuse other init systems. A container - manager may implement the `$NOTIFY_SOCKET` protocol mentioned below in which - case it will receive a notification message `X_SYSTEMD_SIGNALS_LEVEL=2` that - indicates if and when these additional signal handlers are installed. If - these signals are sent to the container's PID 1 before this notification - message is sent they might not be handled correctly yet. - -4. To support [Socket Activated - Containers](https://0pointer.de/blog/projects/socket-activated-containers.html) - the container manager should be capable of being run as a systemd - service. It will then receive the sockets starting with FD 3, the number of - passed FDs in `$LISTEN_FDS` and its PID as `$LISTEN_PID`. It should take - these and pass them on to the container's init process, also setting - $LISTEN_FDS and `$LISTEN_PID` (basically, it can just leave the FDs and - `$LISTEN_FDS` untouched, but it needs to adjust `$LISTEN_PID` to the - container init process). That's all that's necessary to make socket - activation work. The protocol to hand sockets from systemd to services is - hence the same as from the container manager to the container systemd. For - further details see the explanations of - [sd_listen_fds(1)](https://0pointer.de/public/systemd-man/sd_listen_fds.html) - and the [blog story for service - developers](https://0pointer.de/blog/projects/socket-activation.html). - -5. Container managers should stay away from the cgroup hierarchy outside of the - unit they created for their container. That's private property of systemd, - and no other code should modify it. - -6. systemd running inside the container can report when boot-up is complete, - boot progress and functionality as well as various other bits of system - information using the `sd_notify()` protocol that is also used when a - service wants to tell the service manager about readiness. A container - manager can set the `$NOTIFY_SOCKET` environment variable to a suitable - socket path to make use of this functionality. (Also see information about - `/run/host/notify` below, as well as the Readiness Protocol section on - [systemd(1)](https://www.freedesktop.org/software/systemd/man/latest/systemd.html) - -## Networking - -1. Inside of a container, if a `veth` link is named `host0`, `systemd-networkd` - running inside of the container will by default run DHCPv4, DHCPv6, and - IPv4LL clients on it. It is thus recommended that container managers that - add a `veth` link to a container name it `host0`, to get an automatically - configured network, with no manual setup. - -2. Outside of a container, if a `veth` link is prefixed "ve-", `systemd-networkd` - will by default run DHCPv4 and DHCPv6 servers on it, as well as IPv4LL. It - is thus recommended that container managers that add a `veth` link to a - container name the external side `ve-` + the container name. - -3. It is recommended to configure stable MAC addresses for container `veth` - devices, for example, hashed out of the container names. That way it is more - likely that DHCP and IPv4LL will acquire stable addresses. - -## The `/run/host/` Hierarchy - -Container managers may place certain resources the manager wants to provide to -the container payload below the `/run/host/` hierarchy. This hierarchy should -be mostly immutable (possibly some subdirs might be writable, but the top-level -hierarchy — and probably most subdirs should be read-only to the -container). Note that this hierarchy is used by various container managers, and -care should be taken to avoid naming conflicts. `systemd` (and in particular -`systemd-nspawn`) use the hierarchy for the following resources: - -1. The `/run/host/incoming/` directory mount point is configured for `MS_SLAVE` - mount propagation with the host, and is used as intermediary location for - mounts to establish in the container, for the implementation of `machinectl - bind`. Container payload should usually not directly interact with this - directory: it's used by code outside the container to insert mounts inside - it only, and is mostly an internal vehicle to achieve this. Other container - managers that want to implement similar functionality might consider using - the same directory. Alternatively, the new mount API may be used by the - container manager to establish new mounts in the container without the need - for the `/run/host/incoming/` directory. - -2. The `/run/host/inaccessible/` directory may be set up by the container - manager to include six file nodes: `reg`, `dir`, `fifo`, `sock`, `chr`, - `blk`. These nodes correspond with the six types of file nodes Linux knows - (with the exceptions of symlinks). Each node should be of the specific type - and have an all zero access mode, i.e. be inaccessible. The two device node - types should have major and minor of zero (which are unallocated devices on - Linux). These nodes are used as mount source for implementing the - `InaccessiblePath=` setting of unit files, i.e. file nodes to mask this way - are overmounted with these "inaccessible" inodes, guaranteeing that the file - node type does not change this way but the nodes still become - inaccessible. Note that systemd when run as PID 1 in the container payload - will create these nodes on its own if not passed in by the container - manager. However, in that case it likely lacks the privileges to create the - character and block devices nodes (there are fallbacks for this case). - -3. The `/run/host/notify` path is a good choice to place the `sd_notify()` - socket in, that may be used for the container's PID 1 to report to the - container manager when boot-up is complete. The path used for this doesn't - matter much as it is communicated via the `$NOTIFY_SOCKET` environment - variable, following the usual protocol for this, however it's suitable, and - recommended place for this socket in case ready notification is desired. - -4. The `/run/host/os-release` file contains the `/etc/os-release` file of the - host, i.e. may be used by the container payload to gather limited - information about the host environment, on top of what `uname -a` reports. - -5. The `/run/host/container-manager` file may be used to pass the same - information as the `$container` environment variable (see above), i.e. a - short string identifying the container manager implementation. This file - should be newline terminated. Passing this information via this file has the - benefit that payload code can easily access it, even when running - unprivileged without access to the container PID 1's environment block. - -6. The `/run/host/container-uuid` file may be used to pass the same information - as the `$container_uuid` environment variable (see above). This file should - be newline terminated. - -7. The `/run/host/credentials/` directory is a good place to pass credentials - into the container, using the `$CREDENTIALS_DIRECTORY` protocol, see above. - -8. The `/run/host/unix-export/` directory shall be writable from the container - payload, and is where container payload can bind `AF_UNIX` sockets in that - shall be *exported* to the host, so that the host can connect to them. The - container manager should bind mount this directory on the host side - (read-only ideally), so that the host can connect to contained sockets. This - is most prominently used by `systemd-ssh-generator` when run in such a - container to automatically bind an SSH socket into that directory, which - then can be used to connect to the container. - -9. The `/run/host/unix-export/ssh` `AF_UNIX` socket will be automatically bound - by `systemd-ssh-generator` in the container if possible, and can be used to - connect to the container. - -10. The `/run/host/userdb/` directory may be used to drop-in additional JSON - user records that `nss-systemd` inside the container shall include in the - system's user database. This is useful to make host users and their home - directories automatically accessible to containers in transitive - fashion. See `nss-systemd(8)` for details. - -11. The `/run/host/home/` directory may be used to bind mount host home - directories of users that shall be made available in the container to. This - may be used in combination with `/run/host/userdb/` above: one defines the - user record, the other contains the user's home directory. - -## What You Shouldn't Do - -1. Do not drop `CAP_MKNOD` from the container. `PrivateDevices=` is a commonly - used service setting that provides a service with its own, private, minimal - version of `/dev/`. To set this up systemd in the container needs this - capability. If you take away the capability, then all services that set this - flag will cease to work. Use `BPF_PROG_TYPE_CGROUP_DEVICE` BPF programs — on - cgroupv2 — or the `devices` controller — on cgroupv1 — to restrict what - device nodes the container can create instead of taking away the capability - wholesale. (Also see the section about fully unprivileged containers below.) - -2. Do not drop `CAP_SYS_ADMIN` from the container. A number of the most - commonly used file system namespacing related settings, such as - `PrivateDevices=`, `ProtectHome=`, `ProtectSystem=`, `MountFlags=`, - `PrivateTmp=`, `ReadWriteDirectories=`, `ReadOnlyDirectories=`, - `InaccessibleDirectories=`, and `MountFlags=` need to be able to open new - mount namespaces and the mount certain file systems into them. You break all - services that make use of these options if you drop the capability. Also - note that logind mounts `XDG_RUNTIME_DIR` as `tmpfs` for all logged in users - and that won't work either if you take away the capability. (Also see - section about fully unprivileged containers below.) - -3. Do not cross-link `/dev/kmsg` with `/dev/console`. They are different things, - you cannot link them to each other. - -4. Do not pretend that the real VTs are available in the container. The VT - subsystem consists of all the devices `/dev/tty[0-9]*`, `/dev/vcs*`, - `/dev/vcsa*` plus their `sysfs` counterparts. They speak specific `ioctl()`s - and understand specific escape sequences, that other ptys don't understand. - Hence, it is explicitly not OK to mount a pty to `/dev/tty1`, `/dev/tty2`, - `/dev/tty3`. This is explicitly not supported. - -5. Don't pretend that passing arbitrary devices to containers could really work - well. For example, do not pass device nodes for block devices to the - container. Device access (with the exception of network devices) is not - virtualized on Linux. Enumeration and probing of meta information from - `/sys/` and elsewhere is not possible to do correctly in a container. Simply - adding a specific device node to a container's `/dev/` is *not* *enough* to - do the job, as `systemd-udevd` and suchlike are not available at all, and no - devices will appear available or enumerable, inside the container. - -6. Don't mount only a sub-tree of the `cgroupfs` into the container. This will not - work as `/proc/$PID/cgroup` lists full paths and cannot be matched up with - the actual `cgroupfs` tree visible, then. (You may "prune" some branches - though, see above.) - -7. Do not make `/sys/` writable in the container. If you do, - `systemd-udevd.service` is started to manage your devices — inside the - container, but that will cause conflicts and errors given that the Linux - device model is not virtualized for containers on Linux and thus the - containers and the host would try to manage the same devices, fighting for - ownership. Multiple other subsystems of systemd similarly test for `/sys/` - being writable to decide whether to use `systemd-udevd` or assume that - device management is properly available on the instance. Among them - `systemd-networkd` and `systemd-logind`. The conditionalization on the - read-only state of `/sys/` enables a nice automatism: as soon as `/sys/` and - the Linux device model are changed to be virtualized properly the container - payload can make use of that, simply by marking `/sys/` writable. (Note that - as special exception, the devices in `/sys/class/net/` are virtualized - already, if network namespacing is used. Thus it is OK to mount the relevant - sub-directories of `/sys/` writable, but make sure to leave the root of - `/sys/` read-only.) - -8. Do not pass the `CAP_AUDIT_CONTROL`, `CAP_AUDIT_READ`, `CAP_AUDIT_WRITE` - capabilities to the container, in particular not to those making use of user - namespaces. The kernel's audit subsystem is still not virtualized for - containers, and passing these credentials is pointless hence, given the - actual attempt to make use of the audit subsystem will fail. Note that - systemd's audit support is partially conditioned on these capabilities, thus - by dropping them you ensure that you get an entirely clean boot, as systemd - will make no attempt to use it. If you pass the capabilities to the payload - systemd will assume that audit is available and works, and some components - will subsequently fail in various ways. Note that once the kernel learnt - native support for container-virtualized audit, adding the capability to the - container description will automatically make the container payload use it. - -## Fully Unprivileged Container Payload - -First things first, to make this clear: Linux containers are not a security -technology right now. There are more holes in the model than in swiss cheese. - -For example: if you do not use user namespacing, and share root and other users -between container and host, the `struct user` structures will be shared between -host and container, and hence `RLIMIT_NPROC` and so of the container users -affect the host and other containers, and vice versa. This is a major security -hole, and actually is a real-life problem: since Avahi sets `RLIMIT_NPROC` of -its user to 2 (to effectively disallow `fork()`ing) you cannot run more than -one Avahi instance on the entire system... - -People have been asking to be able to run systemd without `CAP_SYS_ADMIN` and -`CAP_SYS_MKNOD` in the container. This is now supported to some level in -systemd, but we recommend against it (see above). If `CAP_SYS_ADMIN` and -`CAP_SYS_MKNOD` are missing from the container systemd will now gracefully turn -off `PrivateTmp=`, `PrivateNetwork=`, `ProtectHome=`, `ProtectSystem=` and -others, because those capabilities are required to implement these options. The -services using these settings (which include many of systemd's own) will hence -run in a different, less secure environment when the capabilities are missing -than with them around. - -With user namespacing in place things get much better. With user namespaces the -`struct user` issue described above goes away, and containers can keep -`CAP_SYS_ADMIN` safely for the user namespace, as capabilities are virtualized -and having capabilities inside a container doesn't mean one also has them -outside. - -## Final Words - -If you write software that wants to detect whether it is run in a container, -please check `/proc/1/environ` and look for the `container=` environment -variable. Do not assume the environment variable is inherited down the process -tree. It generally is not. Hence check the environment block of PID 1, not your -own. Note though that this file is only accessible to root. systemd hence early -on also copies the value into `/run/systemd/container`, which is readable for -everybody. However, that's a systemd-specific interface and other init systems -are unlikely to do the same. - -Note that it is our intention to make systemd systems work flawlessly and -out-of-the-box in containers. In fact, we are interested to ensure that the same -OS image can be booted on a bare system, in a VM and in a container, and behave -correctly each time. If you notice that some component in systemd does not work -in a container as it should, even though the container manager implements -everything documented above, please contact us. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index e9aa5924153..7d0cf991279 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -7,127 +7,36 @@ SPDX-License-Identifier: LGPL-2.1-or-later # Contributing -We welcome contributions from everyone. However, please follow the following guidelines when posting a GitHub Pull Request or filing a GitHub Issue on the systemd project: +We welcome contributions from everyone. Please follow the following guidelines when posting a GitHub Pull Request or filing a GitHub Issue on the FileSystemds project: ## Filing Issues -* We use [GitHub Issues](https://github.com/systemd/systemd/issues) **exclusively** for tracking **bugs** and **feature** **requests** (RFEs) of systemd. - If you are looking for help, please try the forums of your distribution first, or [systemd-devel mailing list](https://lists.freedesktop.org/mailman/listinfo/systemd-devel) for general questions about systemd. -* We only track bugs in the **two** **most** **recently** **released** (non-rc) **versions** of systemd in the GitHub Issue tracker. - If you are using an older version of systemd, please contact your distribution's bug tracker instead (see below). - See [GitHub Release Page](https://github.com/systemd/systemd/releases) for the list of most recent releases. -* When filing a feature request issue (RFE), please always check first if the newest upstream version of systemd already implements the feature, +* We use [GitHub Issues](https://github.com/spiralgang/FileSystemds/issues) **exclusively** for tracking **bugs** and **feature** **requests** (RFEs) of FileSystemds. + If you are looking for help, please check the project documentation or file an issue for general questions. +* When filing a feature request issue (RFE), please always check first if the newest version of FileSystemds already implements the feature, and whether there's already an issue filed for your feature by someone else. -* When filing an issue, specify the **systemd** **version** you are experiencing the issue with. Also, indicate which **distribution** you are using. +* When filing an issue, specify the **FileSystemds** **version** you are experiencing the issue with. Also, indicate which **platform** (Android, desktop, etc.) you are using. * Please include an explanation how to reproduce the issue you are pointing out. Following these guidelines makes it easier for us to process your issue, and ensures we won't close your issue right-away for being misfiled. -### Older downstream versions +## Submitting Pull Requests -For older versions that are still supported by your distribution please use respective downstream tracker: +* Please make sure to run any existing tests and linting before submitting your PR. +* Follow the existing code style and conventions in the project. +* Include a clear description of what your changes do and why they're needed. +* Reference any relevant issues in your PR description. -* **Fedora** - [bugzilla](https://bugzilla.redhat.com/enter_bug.cgi?product=Fedora&component=systemd) -* **RHEL/CentOS stream** - [Jira](https://issues.redhat.com/secure/CreateIssueDetails!init.jspa?pid=12332745&issuetype=1&components=12380515&priority=10300) or [contribute to systemd-rhel @GitHub](https://github.com/redhat-plumbers#systemd) -* **Debian** - [bugs.debian.org](https://bugs.debian.org/cgi-bin/pkgreport.cgi?pkg=systemd) +## Development Guidelines -## Security vulnerability reports +This project follows a modular, agent-driven, mobile/cloud-first architecture. When contributing: -See [reporting of security vulnerabilities](https://systemd.io/SECURITY). +* Prefer modular, composable components over monolithic solutions +* Design for mobile/edge environments +* Consider agent and automation-friendly interfaces +* Include appropriate tests and documentation +* Follow the project's security and quality standards -## Posting Pull Requests +## Code of Conduct -* Make sure to post PRs only relative to a recent tip of the `main` branch. -* Follow our [Coding Style](https://systemd.io/CODING_STYLE) when contributing code. This is a requirement for all code we merge. -* Please make sure to test your change before submitting the PR. See the [Hacking guide](https://systemd.io/HACKING) for details on how to do this. -* Make sure to run the test suite locally, before posting your PR. We use a CI system, meaning we don't even look at a PR if the build fails or the unit tests don't pass. -* If you need to update the code in an existing PR, force-push into the same branch, overriding old commits with new versions. -* After you have pushed a new version, add a comment explaining the latest changes. -* If you are copying existing code from another source (eg: a compat header), please make sure the license is compatible with `LGPL-2.1-or-later`. - If the license is not `LGPL-2.1-or-later`, please add a note to [`LICENSES/README.md`](https://github.com/systemd/systemd/blob/main/LICENSES/README.md). -* If the pull request stalls without review, post a ping in a comment after some time has passed. - We are always short on reviewer time, and pull requests which haven't seen any recent activity can be easily forgotten. -* Github will automatically add the `please-review` label when a pull request is opened or updated. - If you need more information after a review, you can comment `/please-review` on the pull request to have Github add the `please-review` label to the pull request. - -## Using AI Code Generators - -If you use an AI code generator such as ChatGPT, Claude, Copilot, Llama or a similar tool, this must be -disclosed in the commit messages and pull request description. - -The quality bar for contributions to this project is high, and unlikely to be met by an unattended AI tool, -without significant manual corrections. Always thoroughly review and correct any such outputs, for example -ensuring it accurately follows [Coding Style](https://systemd.io/CODING_STYLE) at the very minimum. Please do -not fire-and-forget pull requests without any human intervention and review, as that will likely result in -low-quality results that will not be accepted, and if done repeatedly, may result in the account being -blocked. As with any other submissions, authors are responsible for doing due diligence and ensuring their -submissions are compatible with the project's license as documented in LICENSES/README.md. - -As a guideline, if someone notices that a contribution (code, issues, comments) was made with the help of AI, -there was likely a lack of human review of the AI generated output. - -## Reviewing Pull Requests - -* See [filtered list of pull requests](https://github.com/systemd/systemd/pulls?q=is%3Aopen+is%3Apr+-label%3A%22reviewed%2Fneeds-rework+%F0%9F%94%A8%22+-label%3Aneeds-rebase+-label%3Agood-to-merge%2Fwith-minor-suggestions+-label%3A%22good-to-merge%2Fwaiting-for-ci+%F0%9F%91%8D%22+-label%3Apostponed+-label%3A%22needs-reporter-feedback+%E2%9D%93%22+-label%3A%22dont-merge+%F0%9F%92%A3%22+-label%3A%22ci-fails%2Fneeds-rework+%F0%9F%94%A5%22+sort%3Aupdated-desc) for requests that are ready for review. -* After performing a review, set - - * `reviewed/needs-rework` if the pull request needs significant changes - * `ci-fails/needs-rework` if the automatic tests fail and the failure is relevant to the pull request - * `ci-failure-appears-unrelated` if the test failures seem irrelevant - * `needs-rebase` if the pull request needs a rebase because of conflicts - * `good-to-merge/waiting-for-ci` if the pull request should be merged without further review - * `good-to-merge/with-minor-suggestions` if the pull request should be merged after an update without going through another round of reviews - -Unfortunately only members of the `systemd` organization on github can change labels. -If your pull request is mislabeled, make a comment in the pull request and somebody will fix it. -Reviews from non-members are still welcome. - -## Final Words - -We'd like to apologize in advance if we are not able to process and reply to your issue or PR right-away. We have a lot of work to do, but we are trying our best! - -Thank you very much for your contributions! - -# Backward Compatibility And External Dependencies - -We strive to keep backward compatibility where possible and reasonable. -The following are general guidelines, not hard rules, and case-by-case exceptions might be applied at the discretion of the maintainers. -The current set of build-time and runtime dependencies are documented in the [README](https://github.com/systemd/systemd/blob/main/README). - -## New features - -It is fine for new features/functionality/tools/daemons to require bleeding edge external dependencies, provided there -are runtime and build-time graceful fallbacks (e.g.: a daemon will not be built, runtime functionality will be skipped with a clear log message). -In case a new feature is added to both `systemd` and one of its dependencies, we expect the corresponding feature code to -be merged upstream in the dependency before accepting our side of the implementation. -Making use of new kernel syscalls can be achieved through compat wrappers in our tree (see: `src/basic/missing_syscall_def.h`), -and does not need to wait for glibc support. - -## External Build/Runtime Dependencies - -It is often tempting to bump external dependencies' minimum versions to cut cruft, and in general it's an essential part -of the maintenance process. But as a general rule, existing dependencies should not be bumped without strong -reasons. When possible, we try to keep compatibility with the most recent LTS releases of each mainstream distribution -for optional components, and with all currently maintained (i.e.: not EOL) LTS releases for core components. When in -doubt, ask before committing time to work on contributions if it's not clear that cutting support would be obviously -acceptable. - -## Kernel Requirements - -Same principles as with other dependencies should be applied. It is fine to require newer kernel versions for additional -functionality or optional features, but very strong reasons should be required for breaking compatibility for existing -functionality, especially for core components. It is not uncommon, for example, for embedded systems to be stuck on older -kernel versions due to hardware requirements, so do not assume everybody is running with latest and greatest at all times. -In general, [currently maintained LTS branches](https://www.kernel.org/category/releases.html) should keep being supported -for existing functionality. - -## `libsystemd.so` - -`libsystemd.so` is a shared public library, so breaking ABI/API compatibility would create lot of work for everyone, and is not allowed. -Instead, always add a new interface instead of modifying the signature of an existing function. -It is fine to mark an interface as deprecated to gently nudge users toward a newer one, but support for the old one must be maintained. -Symbol versioning and the compiler's deprecated attribute should be used when managing the lifetime of a public interface. - -## `libudev.so` - -`libudev.so` is a shared public library, and is still maintained, but should not gain new symbols at this point. +Please see our [Code of Conduct](CODE_OF_CONDUCT.md) for community guidelines. \ No newline at end of file diff --git a/docs/CONTROL_GROUP_INTERFACE.md b/docs/CONTROL_GROUP_INTERFACE.md deleted file mode 100644 index f6f6322c75b..00000000000 --- a/docs/CONTROL_GROUP_INTERFACE.md +++ /dev/null @@ -1,244 +0,0 @@ ---- -title: New Control Group Interfaces -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# The New Control Group Interfaces - -> _aka "I want to make use of kernel cgroups, how do I do this in the new world order?"_ - -Starting with version 205 systemd provides a number of interfaces that may be used to create and manage labelled groups of processes for the purpose of monitoring and controlling them and their resource usage. -This is built on top of the Linux kernel Control Groups ("cgroups") facility. - -Previously, the kernel's cgroups API (cgroup v1) was exposed directly as shared application API. That, however, turned out to be undesirable and semantically broken to manage. -The superseding cgroup v2 interface therefore requires that each individual cgroup is managed by a single writer only. - -With this change the main cgroup tree becomes private property of that userspace component and is no longer a shared resource. - -On systemd systems PID 1 takes this role and hence needs to provide APIs for clients to take benefit of the control groups functionality of the kernel. - -Note that services running on systemd systems may manage their own subtrees of the cgroups tree, as long as they explicitly turn on delegation mode for them (see below). - -That means explicitly, that: - -1. The root control group may only be written to by systemd (PID 1). Services that create and manipulate control groups in the top level cgroup are in direct conflict with the kernel's requirement that each control group should have a single-writer only. -2. Services must set Delegate=yes for the units they intend to manage subcgroups of. If they create and manipulate cgroups outside of units that have Delegate=yes set, they violate the access contract for control groups. - -For a more high-level background story, please have a look at this [Linux Foundation News Story](https://www.linuxfoundation.jp/blog/2013/08/all-about-the-linux-kernel-cgroups-redesign/). - -### Why this all again? - -- Objects placed in the same level of the cgroup tree frequently need to propagate properties from one to each other. For example, when using the "cpu" controller for one object then all objects on the same level need to do the same, otherwise the entire cgroup of the first object will be scheduled against the individual processes of the others, thus giving the first object a drastic malus on scheduling if it uses many processes. -- Similar, some properties also require propagation up the tree. -- The tree needs to be refreshed/built in scheduled steps as devices show up/go away as controllers like "blkio" or "devices" refer to devices via major/minor device node indexes, which are not fixed but determined only as a device appears. -- The tree also needs refreshing/rebuilding as new services are installed/started/instantiated/stopped/uninstalled. -- Many of the cgroup attributes are too low-level as API. For example, the major/minor device interface in order to be useful requires a userspace component for translating stable device paths into major/minor at the right time. -- By unifying the cgroup logic under a single arbiter it is possible to write tools that can manage all objects the system contains, including services, virtual machines containers and whatever else applications register. -- By unifying the cgroup logic under a single arbiter a good default that encompasses all kinds of objects may be shipped, thus making manual configuration unnecessary to take benefit of basic resource control. - -systemd through its "unit" concept already implements a dependency network between objects where propagation can take place and contains a powerful execution queue. Also, a major part of the objects resources need to be controlled for are already systemd objects, most prominently the services systemd manages. - -### Why is this not managed by a component independent of systemd? - -Well, as mentioned above, a dependency network between objects, usable for propagation, combined with a powerful execution engine is basically what systemd _is_. Since cgroups management requires precisely this it is an obvious choice to simply implement this in systemd itself. - -Implementing a similar propagation/dependency network with execution scheduler outside of systemd in an independent "cgroup" daemon would basically mean reimplementing systemd a second time. Also, accessing such an external service from PID 1 for managing other services would result in cyclic dependencies between PID 1 which would need this functionality to manage the cgroup service which would only be available however after that service finished starting up. Such cyclic dependencies can certainly be worked around, but make such a design complex. - -### I don't use systemd, what does this mean for me? - -Nothing. This page is about systemd's cgroups APIs. If you don't use systemd then the kernel cgroup rework will probably affect you eventually, but a different component will be the single writer userspace daemon managing the cgroup tree, with different APIs. Note that the APIs described here expose a lot of systemd-specific concepts and hence are unlikely to be available outside of systemd systems. - -### I want to write cgroup code that should work on both systemd systems and others (such as Ubuntu), what should I do? - -On systemd systems use the systemd APIs as described below. At this time we are not aware of any component that would take the cgroup managing role on Upstart/sysvinit systems, so we cannot help you with this. Sorry. - -## systemd's Resource Control Concepts - -Systemd provides three unit types that are useful for the purpose of resource control: - -- [_Services_](http://www.freedesktop.org/software/systemd/man/systemd.service.html) encapsulate a number of processes that are started and stopped by systemd based on configuration. Services are named in the style of `quux.service`. -- [_Scopes_](http://www.freedesktop.org/software/systemd/man/systemd.scope.html) encapsulate a number of processes that are started and stopped by arbitrary processes via fork(), and then registered at runtime with PID1. Scopes are named in the style of `wuff.scope`. -- [_Slices_](http://www.freedesktop.org/software/systemd/man/systemd.slice.html) may be used to group a number of services and scopes together in a hierarchial tree. Slices do not contain processes themselves, but the services and scopes contained in them do. Slices are named in the style of `foobar-waldo.slice`, where the path to the location of the slice in the tree is encoded in the name with "-" as separator for the path components (`foobar-waldo.slice` is hence a subslice of `foobar.slice`). There's one special slices defined, `-.slice`, which is the root slice of all slices (`foobar.slice` is hence subslice of `-.slice`). This is similar how in regular file paths, "/" denotes the root directory. - -Service, scope and slice units directly map to objects in the cgroup tree. When these units are activated they each map to directly (modulo some character escaping) to cgroup paths built from the unit names. For example, a service `quux.service` in a slice `foobar-waldo.slice` is found in the cgroup `foobar.slice/foobar-waldo.slice/quux.service/`. - -Services, scopes and slices may be created freely by the administrator or dynamically by programs. However by default the OS defines a number of built-in services that are necessary to start-up the system. Also, there are four slices defined by default: first of all the root slice `-.slice` (as mentioned above), but also `system.slice`, `machine.slice`, `user.slice`. By default all system services are placed in the first slice, all virtual machines and containers in the second, and user sessions in the third. However, this is just a default, and the administrator my freely define new slices and assign services and scopes to them. Also note that all login sessions automatically are placed in an individual scope unit, as are VM and container processes. Finally, all users logging in will also get an implicit slice of their own where all the session scopes are placed. - -Here's an example how the cgroup tree could look like (as generated with `systemd-cgls(1)`, see below): - -``` -├─user.slice -│ └─user-1000.slice -│ ├─session-18.scope -│ │ ├─703 login -- lennart -│ │ └─773 -bash -│ ├─session-1.scope -│ │ ├─ 518 gdm-session-worker [pam/gdm-autologin] -│ │ ├─ 540 gnome-session -│ │ ├─ 552 dbus-launch --sh-syntax --exit-with-session -│ │ ├─ 553 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session -│ │ ├─ 589 /usr/libexec/gvfsd -│ │ ├─ 593 /usr/libexec//gvfsd-fuse -f /run/user/1000/gvfs -│ │ ├─ 598 /usr/libexec/gnome-settings-daemon -│ │ ├─ 617 /usr/bin/gnome-keyring-daemon --start --components=gpg -│ │ ├─ 630 /usr/bin/pulseaudio --start -│ │ ├─ 726 /usr/bin/gnome-shell -│ │ ├─ 728 syndaemon -i 1.0 -t -K -R -│ │ ├─ 736 /usr/libexec/gsd-printer -│ │ ├─ 742 /usr/libexec/dconf-service -│ │ ├─ 798 /usr/libexec/mission-control-5 -│ │ ├─ 802 /usr/libexec/goa-daemon -│ │ ├─ 823 /usr/libexec/gvfsd-metadata -│ │ ├─ 866 /usr/libexec/gvfs-udisks2-volume-monitor -│ │ ├─ 880 /usr/libexec/gvfs-gphoto2-volume-monitor -│ │ ├─ 886 /usr/libexec/gvfs-afc-volume-monitor -│ │ ├─ 891 /usr/libexec/gvfs-mtp-volume-monitor -│ │ ├─ 895 /usr/libexec/gvfs-goa-volume-monitor -│ │ ├─ 999 /usr/libexec/telepathy-logger -│ │ ├─ 1076 /usr/libexec/gnome-terminal-server -│ │ ├─ 1079 gnome-pty-helper -│ │ ├─ 1080 bash -│ │ ├─ 1134 ssh-agent -│ │ ├─ 1137 gedit l -│ │ ├─ 1160 gpg-agent --daemon --write-env-file -│ │ ├─ 1371 /usr/lib64/firefox/firefox -│ │ ├─ 1729 systemd-cgls -│ │ ├─ 1929 bash -│ │ ├─ 2057 emacs src/login/org.freedesktop.login1.policy.in -│ │ ├─ 2060 /usr/libexec/gconfd-2 -│ │ ├─29634 /usr/libexec/gvfsd-http --spawner :1.5 /org/gtk/gvfs/exec_spaw/0 -│ │ └─31416 bash -│ └─user@1000.service -│ ├─532 /usr/lib/systemd/systemd --user -│ └─541 (sd-pam) -└─system.slice - ├─1 /usr/lib/systemd/systemd --system --deserialize 22 - ├─sshd.service - │ └─29701 /usr/sbin/sshd -D - ├─udisks2.service - │ └─743 /usr/lib/udisks2/udisksd --no-debug - ├─colord.service - │ └─727 /usr/libexec/colord - ├─upower.service - │ └─633 /usr/libexec/upowerd - ├─wpa_supplicant.service - │ └─488 /usr/sbin/wpa_supplicant -u -f /var/log/wpa_supplicant.log -c /etc/wpa_supplicant/wpa_supplicant.conf -u -f /var/log/wpa_supplicant.log -P /var/run/wpa_supplicant.pid - ├─bluetooth.service - │ └─463 /usr/sbin/bluetoothd -n - ├─polkit.service - │ └─443 /usr/lib/polkit-1/polkitd --no-debug - ├─alsa-state.service - │ └─408 /usr/sbin/alsactl -s -n 19 -c -E ALSA_CONFIG_PATH=/etc/alsa/alsactl.conf --initfile=/lib/alsa/init/00main rdaemon - ├─systemd-udevd.service - │ └─253 /usr/lib/systemd/systemd-udevd - ├─systemd-journald.service - │ └─240 /usr/lib/systemd/systemd-journald - ├─rtkit-daemon.service - │ └─419 /usr/libexec/rtkit-daemon - ├─rpcbind.service - │ └─475 /sbin/rpcbind -w - ├─cups.service - │ └─731 /usr/sbin/cupsd -f - ├─avahi-daemon.service - │ ├─417 avahi-daemon: running [delta.local] - │ └─424 avahi-daemon: chroot helper - ├─dbus.service - │ ├─418 /bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation - │ └─462 /usr/sbin/modem-manager - ├─accounts-daemon.service - │ └─416 /usr/libexec/accounts-daemon - ├─systemd-ask-password-wall.service - │ └─434 /usr/bin/systemd-tty-ask-password-agent --wall - ├─systemd-logind.service - │ └─415 /usr/lib/systemd/systemd-logind - ├─ntpd.service - │ └─429 /usr/sbin/ntpd -u ntp:ntp -g - ├─rngd.service - │ └─412 /sbin/rngd -f - ├─libvirtd.service - │ └─467 /usr/sbin/libvirtd - ├─irqbalance.service - │ └─411 /usr/sbin/irqbalance --foreground - ├─crond.service - │ └─421 /usr/sbin/crond -n - ├─NetworkManager.service - │ ├─ 410 /usr/sbin/NetworkManager --no-daemon - │ ├─1066 /sbin/dhclient -d -sf /usr/libexec/nm-dhcp-client.action -pf /var/run/dhclient-enp0s20u2.pid -lf /var/lib/NetworkManager/dhclient-35c8218b-9e45-4b1f-b79e-22334f687340-enp0s20u2.lease -cf /var/lib/NetworkManager/dhclient-enp0s20u2.conf enp0s20u2 - │ └─1070 /sbin/dhclient -d -sf /usr/libexec/nm-dhcp-client.action -pf /var/run/dhclient-enp0s26u1u4i2.pid -lf /var/lib/NetworkManager/dhclient-f404f1ca-ccfe-4957-aead-dec19c126dea-enp0s26u1u4i2.lease -cf /var/lib/NetworkManager/dhclient-enp0s26u1u4i2.conf enp0s26u1u4i2 - └─gdm.service - ├─420 /usr/sbin/gdm - ├─449 /usr/libexec/gdm-simple-slave --display-id /org/gnome/DisplayManager/Displays/_0 - └─476 /usr/bin/Xorg :0 -background none -verbose -auth /run/gdm/auth-for-gdm-pJjwsi/database -seat seat0 -nolisten tcp vt1 -``` - -As you can see, services and scopes contain process and are placed in slices, and slices do not contain processes of their own. Also note that the special "-.slice" is not shown as it is implicitly identified with the root of the entire tree. - -Resource limits may be set on services, scopes and slices the same way. All active service, scope and slice units may easily be viewed with the "systemctl" command. The hierarchy of services and scopes in the slice tree may be viewed with the "systemd-cgls" command. - -Service and slice units may be configured via unit files on disk, or alternatively be created dynamically at runtime via API calls to PID 1. Scope units may only be created at runtime via API calls to PID 1, but not from unit files on disk. Units that are created dynamically at runtime via API calls are called _transient_ units. Transient units exist only during runtime and are released automatically as soon as they finished/got deactivated or the system is rebooted. - -If a service/slice is configured via unit files on disk the resource controls may be configured with the settings documented in [systemd.resource-control(5)](http://www.freedesktop.org/software/systemd/man/systemd.resource-control.html). While the unit are started they may be reconfigured for services/slices/scopes (with changes applying instantly) with the a command line such as: - -``` -# systemctl set-property httpd.service CPUShares=500 MemoryLimit=500M -``` - -This will make these changes persistently, so that after the next reboot they are automatically applied right when the services are first started. By passing the `--runtime` switch the changes can alternatively be made in a volatile way so that they are lost on the next reboot. - -Note that the number of cgroup attributes currently exposed as unit properties is limited. This will be extended later on, as their kernel interfaces are cleaned up. For example cpuset or freezer are currently not exposed at all due to the broken inheritance semantics of the kernel logic. Also, migrating units to a different slice at runtime is not supported (i.e. altering the Slice= property for running units) as the kernel currently lacks atomic cgroup subtree moves. - -(Note that the resource control settings are actually also available on mount, swap and socket units. This is because they may also involve processes run for them. However, normally it should not be necessary to alter resource control settings on these unit types.) - -## The APIs - -Most relevant APIs are exposed via D-Bus, however some _passive_ interfaces are available as shared library, bypassing IPC so that they are much cheaper to call. - -### Creating and Starting - -To create and start a transient (scope, service or slice) unit in the cgroup tree use the `StartTransientUnit()` method on the `Manager` object exposed by systemd's PID 1 on the bus, see the [Bus API Documentation](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.systemd1.html) for details. This call takes four arguments. The first argument is the full unit name you want this unit to be known under. This unit name is the handle to the unit, and is shown in the "systemctl" output and elsewhere. This name must be unique during runtime of the unit. You should generate a descriptive name for this that is useful for the administrator to make sense of it. The second parameter is the mode, and should usually be `replace` or `fail`. The third parameter contains an array of initial properties to set for the unit. It is an array of pairs of property names as string and values as variant. Note that this is an array and not a dictionary! This is that way in order to match the properties array of the `SetProperties()` call (see below). The fourth parameter is currently not used and should be passed as empty array. This call will first create the transient unit and then immediately queue a start job for it. This call returns an object path to a `Job` object for the start job of this unit. - -### Properties - -The properties array of `StartTransientUnit()` may take many of the settings that may also be configured in unit files. Not all parameters are currently accepted though, but we plan to cover more properties with future release. Currently you may set the `Description`, `Slice` and all dependency types of units, as well as `RemainAfterExit`, `ExecStart` for service units, `TimeoutStopUSec` and `PIDs` for scope units, and `CPUAccounting`, `CPUShares`, `BlockIOAccounting`, `BlockIOWeight`, `BlockIOReadBandwidth`, `BlockIOWriteBandwidth`, `BlockIODeviceWeight`, `MemoryAccounting`, `MemoryLimit`, `DevicePolicy`, `DeviceAllow` for services/scopes/slices. These fields map directly to their counterparts in unit files and as normal D-Bus object properties. The exception here is the `PIDs` field of scope units which is used for construction of the scope only and specifies the initial PIDs to add to the scope object. - -To alter resource control properties at runtime use the `SetUnitProperty()` call on the `Manager` object or `SetProperty()` on the individual Unit objects. This also takes an array of properties to set, in the same format as `StartTransientUnit()` takes. Note again that this is not a dictionary, and allows properties to be set multiple times with a single invocation. THis is useful for array properties: if a property is assigned the empty array it will be reset to the empty array itself, however if it is assigned a non-empty array then this array is appended to the previous array. This mimics behaviour of array settings in unit files. Note that most settings may only be set during creation of units with `StartTransientUnit()`, and may not be altered later on. The exception here are the resource control settings, more specifically `CPUAccounting`, `CPUShares`, `BlockIOAccounting`, `BlockIOWeight`, `BlockIOReadBandwidth`, `BlockIOWriteBandwidth`, `BlockIODeviceWeight`, `MemoryAccounting`, `MemoryLimit`, `DevicePolicy`, `DeviceAllow` for services/scopes/slices. Note that the standard D-Bus `org.freedesktop.DBus.Properties.Set()` call is currently not supported by any of the unit objects to set these properties, but might eventually (note however, that it is substantially less useful as it only allows setting a single property at a time, resulting in races). - -The [`systemctl set-property`](http://www.freedesktop.org/software/systemd/man/systemctl.html) command internally is little more than a wrapper around `SetUnitProperty()`. The [`systemd-run`](http://www.freedesktop.org/software/systemd/man/systemd-run.html) tool is a wrapper around `StartTransientUnit()`. It may be used to either run a process as a transient service in the background, where it is invoked from PID 1, or alternatively as a scope unit in the foreground, where it is run from the `systemd-run` process itself. - -### Enumeration - -To acquire a list of currently running units, use the `ListUnits()` call on the Manager bus object. To determine the scope/service unit and slice unit a process is running in use [`sd_pid_get_unit()`](http://www.freedesktop.org/software/systemd/man/sd_pid_get_unit.html) and `sd_pid_get_slice()`. These two calls are implemented in `libsystemd-login.so`. These call bypass the system bus (which they can because they are passive and do not require privileges) and are hence very efficient to invoke. - -### VM and Container Managers - -Use these APIs to register any kind of process workload with systemd to be placed in a resource controlled cgroup. Note however that for containers and virtual machines it is better to use the [`machined`](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.machine1.html) interfaces since they provide integration with "ps" and similar tools beyond what mere cgroup registration provides. Also see [Writing VM and Container Managers](/WRITING_VM_AND_CONTAINER_MANAGERS) for details. - -### Reading Accounting Information - -Accounting information is available via the `MemoryCurrent`, `MemoryPeak`, `MemorySwapCurrent`, `MemorySwapPeak`, `MemoryZSwapCurrent`, `MemoryAvailable`, `EffectiveMemoryMax`, `EffectiveMemoryHigh`, `CPUUsageNSec`, `EffectiveCPUs`, `EffectiveMemoryNodes`, `TasksCurrent`, `EffectiveTasksMax`, `IPIngressBytes`, `IPIngressPackets`, `IPEgressBytes`, `IPEgressPackets`, `IOReadBytes`, `IOReadOperations`, `IOWriteBytes`, and `IOWriteOperations` D-Bus properties. To read this and other information directly from the cgroup tree, get the unit's cgroup path (relative to `/sys/fs/cgroup`) from the `ControlGroup` property, by calling [`sd_pid_get_cgroup()`](https://www.freedesktop.org/software/systemd/man/latest/sd_pid_get_cgroup.html), or by parsing `/proc/$PID/cgroup`. - -If you want to collect the exit status and other runtime parameters of your transient scope or service unit after the processes in them ended set the `RemainAfterExit` boolean property when creating it. This will has the effect that the unit will stay around even after all processes in it died, in the `SubState="exited"` state. Simply watch for state changes until this state is reached, then read the status details from the various properties you need, and finally terminate the unit via `StopUnit()` on the `Manager` object or `Stop()` on the `Unit` object itself. - -### Becoming a Controller - -Optionally, it is possible for a program that registers a scope unit (the "scope manager") for one or more of its child processes to hook into the shutdown logic of the scope unit. Normally, if this is not done, and the scope needs to be shut down (regardless if during normal operation when the user invokes `systemctl stop` -- or something equivalent -- on the scope unit, or during system shutdown), then systemd will simply send SIGTERM to its processes. After a timeout this will be followed by SIGKILL unless the scope processes exited by then. If a scope manager program wants to be involved in the shutdown logic of its scopes it may set the `Controller` property of the scope unit when creating it via `StartTransientUnit()`. It should be set to the bus name (either unique name or well-known name) of the scope manager program. If this is done then instead of SIGTERM to the scope processes systemd will send the RequestStop() bus signal to the specified name. If the name is gone by then it will automatically fallback to SIGTERM, in order to make this robust. As before in either case this will be followed by SIGKILL to the scope unit processes after a timeout. - -Scope units implement a special `Abandon()` method call. This method call is useful for informing the system manager that the scope unit is no longer managed by any scope manager process. Among other things it is useful for manager daemons which terminate but want to leave the scopes they started running. When a scope is abandoned its state will be set to "abandoned" which is shown in the usual systemctl output, as information to the user. Also, if a controller has been set for the scope, it will be unset. Note that there is not strictly need to ever invoke the `Abandon()` method call, however it is recommended for cases like the ones explained above. - -### Delegation - -Service and scope units know a special `Delegate` boolean property. If set, then the processes inside the scope or service may control their own control group subtree (that means: create subcgroups directly via /sys/fs/cgroup). The effect of the property is that: - -1. All controllers systemd knows are enabled for that scope/service, if the scope/service runs privileged code. -2. Access to the cgroup directory of the scope/service is permitted, and files/and directories are updated to get write access for the user specified in `User=` if the scope/unit runs unprivileged. Note that in this case access to any controllers is not available. -3. systemd will refrain from moving processes across the "delegation" boundary. - -Generally, the `Delegate` property is only useful for services that need to manage their own cgroup subtrees, such as container managers. After creating a unit with this property set, they should use `/proc/$PID/cgroup` to figure out the cgroup subtree path they may manage (the one from the name=systemd hierarchy!). Managers should refrain from making any changes to the cgroup tree outside of the subtrees for units they created with the `Delegate` flag turned on. - -Note that scope units created by `machined`'s `CreateMachine()` call have this flag set. - -### Example - -Please see the [systemd-run sources](https://github.com/systemd/systemd/blob/main/src/run/run.c) for a relatively simple example how to create scope or service units transiently and pass properties to them. diff --git a/docs/CONVERTING_TO_HOMED.md b/docs/CONVERTING_TO_HOMED.md deleted file mode 100644 index a31ff5a37e5..00000000000 --- a/docs/CONVERTING_TO_HOMED.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -title: Converting Existing Users to systemd-homed -category: Users, Groups and Home Directories -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Converting Existing Users to systemd-homed managed Users - -Traditionally on most Linux distributions, regular (human) users are managed -via entries in `/etc/passwd`, `/etc/shadow`, `/etc/group` and `/etc/gshadow`. -With the advent of -[`systemd-homed`](https://www.freedesktop.org/software/systemd/man/systemd-homed.service.html) -it might be desirable to convert an existing, traditional user account to a -`systemd-homed` managed one. -Below is a brief guide how to do that. - -Before continuing, please read up on these basic concepts: - -* [Home Directories](/HOME_DIRECTORY) -* [JSON User Records](/USER_RECORD) -* [JSON Group Records](/GROUP_RECORD) -* [User/Group Record Lookup API via Varlink](/USER_GROUP_API) - -## Caveat - -This is a manual process, and possibly a bit fragile. -Hence, do this at your own risk, read up beforehand, and make a backup first. -You know what's at stake: your own home directory, i.e. all your personal data. - -## Step-By-Step - -Here's the step-by-step guide: - -0. Preparations: make sure you run a distribution that has `systemd-homed` - enabled and properly set up, including the necessary PAM and NSS configuration updates. - Make sure you have enough disk space in `/home/` for a (temporary) second copy of your home directory. - Make sure to backup your home directory. - Make sure to log out of your user account fully. - Then log in as root on the console. - -1. Rename your existing home directory to something safe. Let's say your user - ID is `foobar`. Then do: - - ``` - mv /home/foobar /home/foobar.saved - ``` - -2. Have a look at your existing user record, as stored in `/etc/passwd` and related files. - We want to use the same data for the new record, hence it's good looking at the old data. - - Use commands such as: - - ``` - getent passwd foobar - getent shadow foobar - ``` - - This will tell you the `/etc/passwd` and `/etc/shadow` entries for your user. - For details about the fields, see the respective man pages - [passwd(5)](https://man7.org/linux/man-pages/man5/passwd.5.html) and - [shadow(5)](https://man7.org/linux/man-pages/man5/shadow.5.html). - - The fourth field in the `getent passwd foobar` output tells you the GID of your user's main group. - Depending on your distribution it's a group private to the user, or a group shared by most local, regular users. - Let's say the GID reported is 1000, let's then query its details: - - ``` - getent group 1000 - ``` - - This will tell you the name of that group. - If the name is the same as your user name your distribution apparently provided you with a private group for your user. - If it doesn't match (and is something like `users`) it apparently didn't. - Note that `systemd-homed` will always manage a private group for each user under the same name, - hence if your distribution is one of the latter kind, then there's a (minor) mismatch in structure when converting. - - Save the information reported by these three commands somewhere, for later reference. - -3. Now edit your `/etc/passwd` file and remove your existing record - (i.e. delete a single line, the one of your user's account, leaving all other lines unmodified). - Similar for `/etc/shadow`, `/etc/group` (in case you have a private group for your user) and `/etc/gshadow`. - Most distributions provide you with a tool for that, that adds safe - synchronization for these changes: `vipw`, `vipw -s`, `vigr` and `vigr -s`. - -4. At this point the old user account vanished, while the home directory still - exists safely under the `/home/foobar.saved` name. - Let's now create a new account with `systemd-homed`, using the same username and UID as before: - - ```sh - homectl create foobar --uid=$UID --real-name=$GECOS - ``` - - In this command line, replace `$UID` by the UID you previously used, - i.e. the third field of the `getent passwd foobar` output above. - Similar, replace `$GECOS` by the GECOS field of your old account, i.e the fifth field of the old output. - If your distribution traditionally does not assign a private group to regular user groups, - then consider adding `--member-of=` with the group name to get a modicum of compatibility with the status quo ante: - this way your new user account will still not have the old primary - group as new primary group, but will have it as auxiliary group. - - Consider reading through the - [homectl(1)](https://www.freedesktop.org/software/systemd/man/homectl.html) - manual page at this point, maybe there are a couple of other settings you want to set for your new account. - In particular, look at `--storage=` and `--disk-size=`, in order to change how your home directory shall be stored - (the default `luks` storage is recommended). - -1. Your new user account exists now, but it has an empty home directory. - Let's now migrate your old home directory into it. - For that let's mount the new home directory temporarily and copy the data in. - - ``` - homectl with foobar -- rsync -aHANUXv --remove-source-files /home/foobar.saved/ . - ``` - - This mounts the home directory of the user, and then runs the specified - `rsync` command which copies the contents of the old home directory into the new. - The new home directory is the working directory of the invoked `rsync` process. - We are invoking this command as root, hence the `rsync` runs as root too. - When the `rsync` command completes the home directory is automatically unmounted again. - Since we used `--remove-source-files` all files copied are removed from the old home directory as the copy progresses. - After the command completes the old home directory should be empty. - Let's remove it hence: - - ``` - rmdir /home/foobar.saved - ``` - -And that's it, we are done already. -You can log out now and should be able to log in under your user account as usual, -but now with `systemd-homed` managing your home directory. diff --git a/docs/DAEMON_SOCKET_ACTIVATION.md b/docs/DAEMON_SOCKET_ACTIVATION.md deleted file mode 100644 index 107615e2ea8..00000000000 --- a/docs/DAEMON_SOCKET_ACTIVATION.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -title: Socket Activation with Popular Daemons -category: Manuals and Documentation for Users and Administrators -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -## nginx - -nginx includes an undocumented, internal socket-passing mechanism based on the `NGINX` environmental variable. -It uses this to perform reloads without having to close and reopen its sockets, but it's also useful for socket activation. - -**/etc/nginx/my-nginx.conf** - -``` -http { - server { - listen [::]:80 ipv6only=on; - listen 80; - } -} -``` - -**/etc/systemd/system/my-nginx.service** - -``` -[Service] -User=nginx -Group=nginx -Environment=NGINX=3:4; -ExecStart=/usr/sbin/nginx -c/etc/nginx/my-nginx.conf -PrivateNetwork=true -``` - -**/etc/systemd/system/my-nginx.socket** - -``` -[Socket] -ListenStream=80 -ListenStream=0.0.0.0:80 -BindIPv6Only=ipv6-only - -[Install] -WantedBy=sockets.target -``` - -## PHP-FPM - -Like nginx, PHP-FPM includes a socket-passing mechanism an environmental variable. -In PHP-FPM's case, it's `FPM_SOCKETS`. - -This configuration is possible with any web server that supports FastCGI (like Apache, Lighttpd, or nginx). -The web server does not need to know anything special about the socket; use a normal PHP-FPM configuration. - -Paths are based on a Fedora 19 system. - -### First, the configuration files - -**/etc/php-fpm.d/my-php-fpm-pool.conf** - -``` -[global] -pid = /run/my-php-fpm-pool.pid ; Not really used by anything with daemonize = no, but needs to be writable. -error_log = syslog ; Will aggregate to the service's systemd journal. -daemonize = no ; systemd handles the forking. - -[www] -listen = /var/run/my-php-fpm-pool.socket ; Must match systemd socket unit. -user = nginx ; Ignored but required. -group = nginx ; Ignored but required. -pm = static -pm.max_children = 10 -slowlog = syslog -``` - -**/etc/systemd/system/my-php-fpm-pool.service** - -``` -[Service] -User=nginx -Group=nginx -Environment="FPM_SOCKETS=/var/run/my-php-fpm-pool.socket=3" -ExecStart=/usr/sbin/php-fpm --fpm-config=/etc/php-fpm.d/my-php-fpm-pool.conf -KillMode=process -``` - -**/etc/systemd/system/my-php-fpm-pool.socket** - -``` -[Socket] -ListenStream=/var/run/my-php-fpm-pool.socket - -[Install] -WantedBy=sockets.target -``` - -### Second, the setup commands - -```sh -sudo systemctl --system daemon-reload -sudo systemctl start my-php-fpm-pool.socket -sudo systemctl enable my-php-fpm-pool.socket -``` - -After accessing the web server, the service should be running. - -```sh -sudo systemctl status my-php-fpm-pool.service -``` - -It's possible to shut down the service and re-activate it using the web browser, too. -It's necessary to stop and start the socket to reset some shutdown PHP-FPM does that otherwise breaks reactivation. - -```sh -sudo systemctl stop my-php-fpm-pool.socket my-php-fpm-pool.service -sudo systemctl start my-php-fpm-pool.socket -``` diff --git a/docs/DISCOVERABLE_PARTITIONS.md b/docs/DISCOVERABLE_PARTITIONS.md deleted file mode 100644 index bc05b6cc5a4..00000000000 --- a/docs/DISCOVERABLE_PARTITIONS.md +++ /dev/null @@ -1 +0,0 @@ -[This content has moved to the UAPI group website](https://uapi-group.org/specifications/specs/discoverable_partitions_specification/) diff --git a/docs/DISTRO_PORTING.md b/docs/DISTRO_PORTING.md deleted file mode 100644 index df9261c27fb..00000000000 --- a/docs/DISTRO_PORTING.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -title: Porting systemd To New Distributions -category: Concepts -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Porting systemd To New Distributions - -## HOWTO - -You need to make the follow changes to adapt systemd to your distribution: - -1. Find the right configure parameters for: - - * `-Dsysvinit-path=` - * `-Dsysvrcnd-path=` - * `-Drc-local=` - * `-Dloadkeys-path=` - * `-Dsetfont-path=` - * `-Dtty-gid=` - * `-Dntp-servers=` - * `-Ddns-servers=` - * `-Dsupport-url=` - -2. Try it out. - - Play around (as an ordinary user) with - `/usr/lib/systemd/systemd --test --system` for a test run of systemd without booting. - This will read the unit files and print the initial transaction it would execute during boot-up. - This will also inform you about ordering loops and suchlike. - -## Compilation options - -The default configuration does not enable any optimization or hardening options. -This is suitable for development and testing, but not for end-user -installations. - -For deployment, optimization (`-O2` or `-O3` compiler options), link time -optimization (`-Db_lto=true` meson option), and hardening (e.g. -`-D_FORTIFY_SOURCE=2`, `-fstack-protector-strong`, `-fstack-clash-protection`, -`-fcf-protection`, `-pie` compiler options, and `-z relro`, `-z now`, -`--as-needed` linker options) are recommended. -The most appropriate set of options depends on the architecture and distribution specifics so no default is -provided. - -## NTP Pool - -By default, systemd-timesyncd uses the Google Public NTP servers -`time[1-4].google.com`, if no other NTP configuration is available. -They serve time that uses a -[leap second smear](https://developers.google.com/time/smear) -and can be up to .5s off from servers that use stepped leap seconds. - -If you prefer to use leap second steps, please register your own -vendor pool at ntp.org and make it the built-in default by -passing `-Dntp-servers=` to meson. -Registering vendor pools is -[free](http://www.pool.ntp.org/en/vendors.html). - -Use `-Dntp-servers=` to direct systemd-timesyncd to different fallback -NTP servers. - -## DNS Servers - -By default, systemd-resolved uses Cloudflare, Google, Quad9 and DNS0 Public DNS servers -`1.1.1.1`, `8.8.8.8`, `9.9.9.9`, `193.110.81.0`, `1.0.0.1`, `8.8.4.4`, `149.112.112.112`, `185.253.5.0`, `2606:4700:4700::1111`, `2001:4860:4860::8888`, `2620:fe::fe`, `2a0f:fc80::`, `2606:4700:4700::1001`, `2001:4860:4860::8844`, `2620:fe::9`, `2a0f:fc81::` -as fallback, if no other DNS configuration is available. - -Use `-Ddns-servers=` to direct systemd-resolved to different fallback -DNS servers. - -## PAM - -The default PAM config shipped by systemd is really bare bones. -It does not include many modules your distro might want to enable -to provide a more seamless experience. -For example, limits set in `/etc/security/limits.conf` will not be read unless you load `pam_limits`. -Make sure you add modules your distro expects from user services. - -Pass `-Dpamconfdir=no` to meson to avoid installing this file and -instead install your own. - -## Contributing Upstream - -We generally no longer accept distribution-specific patches to -systemd upstream. -If you have to make changes to systemd's source code to make it work on your distribution, unless your code is generic enough to be generally useful, we are unlikely to merge it. -Please always consider adopting the upstream defaults. -If that is not possible, please maintain the relevant patches downstream. - -Thank you for understanding. diff --git a/docs/FACTORY_RESET.md b/docs/FACTORY_RESET.md deleted file mode 100644 index bb0e3063b11..00000000000 --- a/docs/FACTORY_RESET.md +++ /dev/null @@ -1,137 +0,0 @@ ---- -title: Factory Reset -category: Booting -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Factory Reset - -In various scenarios it is important to be able to reset operating systems back -into a "factory state", i.e. where all state, user data and configuration is -reset so that it resembles the system state when it was originally shipped. - -systemd natively supports a concept of factory reset, that can both act as a -specific implementation for UEFI based systems, as well as a series of hook -points and a template for implementations on other systems. - -Factory reset always takes place during early boot, i.e. from a well-defined -"clean" state. Factory reset operations may be requested from one boot to be -executed on the next. - -Specifically, the following concepts are available: - -* The `factory-reset.target` unit may be used to request a factory reset - operation and trigger a reboot in order to execute it. It by default executes - three services: `systemd-factory-reset-request.service`, - `systemd-tpm2-clear.service` and `systemd-factory-reset-reboot.service`. - -* The - [`systemd-factory-reset-request.service`](https://www.freedesktop.org/software/systemd/man/latest/systemd-factory-reset-request.service.html) - unit is typically invoked via `factory-reset.target`. It requests a factory - reset operation for the next boot by setting the `FactoryResetRequest` EFI - variable. The EFI variable contains information about the requesting OS, so - that multi-boot scenarios are somewhat covered. - -* The - [`systemd-tpm2-clear.service`](https://www.freedesktop.org/software/systemd/man/latest/systemd-tpm2-clear.service.html) - unit can request a TPM2 clear operation from the firmware on the next - boot. It is also invoked via `factory-reset.target`. UEFI firmwares that - support TPMs will ask the user for confirmation and then reset the TPM, - invalidating all prior keys associated with the security chip and generating - a new seed key. - -* The - [`systemd-factory-reset-reboot.service`](https://www.freedesktop.org/software/systemd/man/latest/systemd-factory-reset-reboot.service.html) - unit automatically reboots the system as part of `factory-reset.target`. It - is ordered after `systemd-tpm2-clear.service` and - `systemd-factory-reset-request.service` in order to initiate the reboot that - is supposed to execute the factory reset operations. - -* The `factory-reset-now.target` unit is started at boot whenever a factory - reset is requested for the boot. A factory reset may be requested via a - kernel command line option (`systemd.factory_reset=1`) or via the UEFI - variable `FactoryResetRequest` (see above). The - `systemd-factory-reset-generator` unit generator checks both these conditions - and adds `factory-reset-now.target` to the boot transaction, already in the - initial RAM disk (initrd). - -* The - [`systemd-factory-reset-complete.service`](https://www.freedesktop.org/software/systemd/man/latest/systemd-factory-reset-complete.service.html) - unit is invoked after `factory-reset-now.target` and marks the factory reset - operation as complete. The boot process then may continue. - -* The - [`systemd-repart`](https://www.freedesktop.org/software/systemd/man/latest/systemd-repart.html) - tool can take the factory reset logic into account. Either on explicit - request via the `--factory-reset=` logic, or automatically derived from the - aforementioned kernel command line switch and EFI variable. When invoked for - factory reset it will securely erase all partitions marked for that via the - `FactoryReset=` setting in its partition definition files. Once that is - complete it will execute the usual setup operation, i.e. format new - partitions again. - -* The - [`systemd-logind.service(8)`](https://www.freedesktop.org/software/systemd/man/latest/systemd-logind.service.html) - unit supports automatically binding factory reset to special keypresses - (typically long presses), see the - [`logind.conf(5)`](https://www.freedesktop.org/software/systemd/man/latest/logind.conf.html) - man page. - -* The - [`systemd-factory-reset`](https://www.freedesktop.org/software/systemd/man/latest/systemd-factory-reset.html) - tool can be used to query the current state of the factory request mechanism, - i.e. whether a factory reset is currently being executed, or if one has been - requested for the next boot. - -* The `/run/systemd/io.systemd.FactoryReset` Varlink service provides two IPC - APIs for working with factory reset: it permits querying whether the local - system supports requesting a factory reset by starting - `factory-reset.target`. This may be used by UIs to hide or show in the UI an - interface to request a factory reset. The Varlink IPC service also reports - the current factory reset state, much like the `systemd-factory-reset` tool - mentioned above. This may be used by various early boot services that - potentially intent to reset system state during a factory reset operation. - -## Exposure in the UI - -If a graphical UI shall expose a factory reset operation it should first check -if requesting a factory reset is supported at all via the Varlink service -mentioned above. Once a factory reset shall be executed it shall ask for -activation of the `factory-reset.target` unit. - -Alternatively, `systemd-logind.service`'s hotkey support may be used, for -example to request factory reset if the reboot button is pressed for a long -time. - -## Support for non-UEFI Systems - -The above is a relatively bespoke solution for EFI systems. It uses EFI -variables as stateful memory to request the factory reset on the next boot. - -On non-EFI systems, a different mechanism should be devised. A service -requesting the factory request can then be plugged into -`factory-reset.target`. At boot the request should then be fed back to the -booted kernel via the `systemd.factory_reset=1` kernel command line option, in -order to execute the reset operation. - -## Support for Resetting other Resources than Partitions + TPM - -By default a factory reset implemented with systemd's tools can reset/erase -partitions (via `systemd-repart`, see above) and reset the TPM (via -`systemd-tpm2-clear.service`, see above). - -In some cases other resources shall be reset/erased too. To support that, -define your own service and plug it into `factory-reset-now.target`, ensuring -it is ordered before that. - -## Factory Reset via Boot Menu - -Factory reset can also be requested via the boot menu. A simple factory reset -(that does not touch the TPM) at boot can be requested via a boot menu item -containing the `systemd.factory_reset=1` kernel command line option. A more -comprehensive factory reset operation (that also erases the TPM) can be -requested by booting with `rd.systemd.unit=factory-reset.target`. Note that the -latter will require one reboot (required since that's how TPM resets work), -while the former will reset state and continue running without an additional -reboot. diff --git a/docs/HACKING.md b/docs/HACKING.md index 37f1148b54b..fdb3cdbf380 100644 --- a/docs/HACKING.md +++ b/docs/HACKING.md @@ -1,341 +1,73 @@ --- -title: Hacking on systemd +title: Hacking on FileSystemds category: Contributing layout: default SPDX-License-Identifier: LGPL-2.1-or-later --- -# Hacking on systemd +# Hacking on FileSystemds -We welcome all contributions to systemd. If you notice a bug or a missing +We welcome all contributions to FileSystemds Mobile Platform. If you notice a bug or a missing feature, please feel invited to fix it, and submit your work as a -[GitHub Pull Request (PR)](https://github.com/systemd/systemd/pull/new). +[GitHub Pull Request (PR)](https://github.com/spiralgang/FileSystemds/pull/new). -Please make sure to follow our [Coding Style](/CODING_STYLE) when submitting -patches. Also have a look at our [Contribution Guidelines](/CONTRIBUTING). +Please make sure to follow our [Coding Style](CODING_STYLE.md) when submitting +patches. Also have a look at our [Contribution Guidelines](CONTRIBUTING.md). -When adding new functionality, tests should be added. For shared functionality -(in `src/basic/` and `src/shared/`) unit tests should be sufficient. The general -policy is to keep tests in matching files underneath `src/test/`, e.g. -`src/test/test-path-util.c` contains tests for any functions in -`src/basic/path-util.c`. If adding a new source file, consider adding a matching -test executable. For features at a higher level, tests in `src/test/` are very -strongly recommended. If that is not possible, integration tests in `test/` are -encouraged. Please always test your work before submitting a PR. +## Development Principles -## Hacking on systemd with mkosi +FileSystemds follows a modular, agent-driven, mobile/cloud-first architecture. When hacking on the project: -[mkosi](https://mkosi.systemd.io/) is our swiss army knife for hacking on -systemd. It makes sure all necessary dependencies are available to build systemd -and allows building and booting an OS image with the latest systemd installed -for testing purposes. +- **Design for modularity**: Create replaceable components, not hardwired logic +- **Think agent-first**: APIs and workflows should be automation-friendly +- **Mobile/edge focus**: Consider mobile and edge computing constraints +- **Security by design**: Follow pointer-first artifact management and security best practices -First, install `mkosi` from the -[GitHub repository](https://github.com/systemd/mkosi#running-mkosi-from-the-repository) -or via your distribution's package manager. Note that systemd regularly adopts -newer mkosi features that are not in an official release yet so there's a good -chance that your distribution's packaged version of mkosi will be too old. +## Testing -Then, you can build, run and test systemd executables as follows: +When adding new functionality, tests should be added. Please always test your work before submitting a PR. -```sh -$ mkosi -f genkey # Generate signing keys once. -$ mkosi -f box -- meson setup -Dbpf-framework=disabled build # bpftool detection inside mkosi box is broken on Ubuntu Noble and older -$ mkosi -f box -- meson compile -C build -$ mkosi -f box -- build/systemctl --version -$ mkosi -f box -- meson test -C build # Run the unit tests -``` - -To build and boot an OS image with the latest systemd installed: - -```sh -$ mkosi -f box -- meson compile -C build mkosi # (re-)build the OS image -$ mkosi boot # Boot the image with systemd-nspawn. -$ mkosi vm # Boot the image with qemu. -``` - -Putting this all together, here's a series of commands for preparing a patch for -systemd: - -```sh -$ git clone https://github.com/systemd/mkosi.git -$ ln -s $PWD/mkosi/bin/mkosi ~/.local/bin/mkosi # Make sure ~/.local/bin is in $PATH. -$ git clone https://github.com/systemd/systemd.git -$ cd systemd -$ git checkout -b # where BRANCH is the name of the branch -$ $EDITOR src/core/main.c # or wherever you'd like to make your changes -$ mkosi -f genkey # Generate signing keys once. -$ mkosi -f box -- meson setup build # Set up meson -$ mkosi -f box -- meson compile -C build mkosi # (re-)build the test image -$ mkosi vm # Boot the image in qemu -$ git add -p # interactively put together your patch -$ git commit # commit it -$ git push -u # where REMOTE is your "fork" on GitHub -``` - -And after that, head over to your repo on GitHub and click "Compare & pull -request" - -Happy hacking! - -The following sections contain advanced topics on how to speed up development or -streamline debugging. Feel free to read them if you're interested but they're -not required to write basic patches. - -## Building the OS image without a tools tree - -By default, `mkosi` will first build a tools tree and use it build the image and -provide the environment for `mkosi box`. To disable the tools tree and use -binaries from your host instead, write the following to `mkosi/mkosi.local.conf`: - -```conf -[Build] -ToolsTree= -``` - -## Rebuilding systemd without rebuilding the OS image - -Every time the `mkosi` target is built, a fresh image is built. To build the -latest changes and re-install systemd without rebuilding the image, run one of -the following commands in another terminal on your host after booting the image -(choose the right one depending on the distribution of the container or virtual -machine): - -```sh -mkosi -R && mkosi ssh -- dnf upgrade --disablerepo="*" --assumeyes "/work/build/*.rpm" # CentOS/Fedora -mkosi -R && mkosi ssh -- apt-get install "/work/build/*.deb" # Debian/Ubuntu -mkosi -R && mkosi ssh -- pacman --upgrade --needed --noconfirm "/work/build/*.pkg.tar" # Arch Linux -mkosi -R && mkosi ssh -- zypper --non-interactive install --allow-unsigned-rpm "/work/build/*.rpm" # OpenSUSE -``` - -and optionally restart the daemon(s) you're working on using -`systemctl restart ` or `systemctl daemon-reexec` if you're working on -pid1 or `systemctl soft-reboot` to restart everything. - -## Building distribution packages with mkosi - -To build distribution packages for a specific distribution and release without -building an actual image, the following command can be used: - -```sh -mkosi -d -r -t none -f -``` - -Afterwards the distribution packages will be located in -`build/mkosi.builddir/~~/`. To also build -debuginfo packages, the following command can be used: - -```sh -mkosi -d -r -E WITH_DEBUG=1 -t none -f -``` - -To upgrade the systemd packages on the host system to the newer versions built -by mkosi, run the following: - -```sh -run0 dnf upgrade build/mkosi.builddir/~~/*.rpm # Fedora/CentOS -run0 apt-get install build/mkosi.builddir/~~/*.deb # Debian/Ubuntu -run0 pacman --upgrade --needed --noconfirm build/mkosi.builddir/~~/*.pkg.tar # Arch Linux -run0 zypper --non-interactive install --allow-unsigned-rpm build/mkosi.builddir/~~/*.rpm # OpenSUSE -``` - -To downgrade back to the old version shipped by the distribution, run the -following: - -```sh -run0 dnf downgrade "systemd*" # Fedora/CentOS -# TODO: Other distributions -``` - -## Installing packages built from the main branch - -Packages for main distributions are built on the SUSE Open Build Service and -repositories are published, so that they can be installed and upgraded easily. +For mobile platform development: +- Test on actual Android devices when possible +- Consider different screen sizes and orientations +- Test offline capabilities +- Verify resource usage is appropriate for mobile devices -Instructions on how to add the repository for each supported distribution can -[be found on OBS.](https://software.opensuse.org//download.html?project=system%3Asystemd&package=systemd) -The `systemd-boot` file is signed for Secure Boot, the self-signed certificate -can be downloaded for enrollment. For example, when using MOK Manager: +## Building and Testing -```sh -$ wget https://build.opensuse.org/projects/system:systemd/signing_keys/download?kind=ssl -O- | openssl x509 -inform pem -outform der -out obs.der -$ run0 mokutil --import obs.der -``` - -## Templating engines in .in files - -Some source files are generated during build. We use two templating engines: -* meson's `configure_file()` directive uses syntax with `@VARIABLE@`. - -See the [Meson docs for `configure_file()`](https://mesonbuild.com/Reference-manual.html#configure_file) for details. - -{% raw %} -* most files are rendered using jinja2, with `{{VARIABLE}}` and `{% if … %}`, -`{% elif … %}`, `{% else … %}`, `{% endif … %}` blocks. `{# … #}` is a jinja2 comment, -i.e. that block will not be visible in the rendered output. -`{% raw %} … `{% endraw %}`{{ '{' }}{{ '% endraw %' }}}` creates a block where jinja2 syntax is not interpreted. - -See the [Jinja Template Designer Documentation](https://jinja.palletsprojects.com/en/3.1.x/templates/#synopsis) for details. - -Please note that files for both template engines use the `.in` extension. - -## Developer and release modes - -In the default meson configuration (`-Dmode=developer`), -certain checks are enabled that are suitable when hacking on systemd (such as internal documentation consistency checks). -Those are not useful when compiling for distribution and can be disabled by setting `-Dmode=release`. - -## Sanitizers in mkosi - -See [Testing systemd using sanitizers](/TESTING_WITH_SANITIZERS) for more information on how to build with sanitizers enabled in mkosi. - -## Debugging binaries that need to run as root in vscode - -When trying to debug binaries that need to run as root, -we need to do some custom configuration in vscode to have it try to run the applications as root and to ask the user for the root password when trying to start the binary. -To achieve this, we'll use a custom debugger path which points to a script that starts `gdb` as root using `pkexec`. -pkexec will prompt the user for their root password via a graphical interface. -This guide assumes the C/C++ extension is used for debugging. - -First, create a file `sgdb` in the root of the systemd repository with the following contents and make it executable: - -```sh -#!/bin/sh -exec pkexec gdb "$@" -``` - -Then, open launch.json in vscode, and set `miDebuggerPath` to `${workspaceFolder}/sgdb` for the corresponding debug configuration. -Now, whenever you try to debug the application, vscode will try to start gdb as root via pkexec which will prompt you for your password via a graphical interface. -After entering your password, vscode should be able to start debugging the application. +### Android APK Build -For more information on how to set up a debug configuration for C binaries, -please refer to the official vscode documentation [here](https://code.visualstudio.com/docs/cpp/launch-json-reference) +The project includes automated Android APK building. To test locally: -## Debugging systemd with mkosi + vscode - -To simplify debugging systemd when testing changes using mkosi, we're going to show how to attach [VSCode](https://code.visualstudio.com/)'s debugger to an instance of systemd running in a mkosi image using QEMU. - -To allow VSCode's debugger to attach to systemd running in a mkosi image, -we have to make sure it can access the virtual machine spawned by mkosi where systemd is running. -After booting the image with `mkosi vm`, -you should now be able to connect to it by running `mkosi ssh` from the same directory in another terminal window. - -Now we need to configure VSCode. -First, make sure the C/C++ extension is installed. -If you're already using a different extension for code completion and other IDE features for C in VSCode, -make sure to disable the corresponding parts of the C/C++ extension in your VSCode user settings by adding the following entries: - -```json -"C_Cpp.formatting": "Disabled", -"C_Cpp.intelliSenseEngine": "Disabled", -"C_Cpp.enhancedColorization": "Disabled", -"C_Cpp.suggestSnippets": false, +```bash +# Check the GitHub Actions workflow for build steps +.github/workflows/android-apk-build.yml ``` -With the extension set up, -we can create the launch.json file in the .vscode/ directory to tell the VSCode debugger how to attach to the systemd instance running in our mkosi container/VM. -Create the file, and possibly the directory, and add the following contents: +### Running Tests -```json -{ - "version": "0.2.0", - "configurations": [ - { - "type": "cppdbg", - "program": "/usr/lib/systemd/systemd", - "processId": "${command:pickRemoteProcess}", - "request": "attach", - "name": "systemd", - "pipeTransport": { - "pipeProgram": "mkosi", - "pipeArgs": ["-C", "${workspaceFolder}", "ssh"], - "debuggerPath": "/usr/bin/gdb" - }, - "MIMode": "gdb", - "sourceFileMap": { - "/work/src": { - "editorPath": "${workspaceFolder}", - "useForBreakpoints": false - }, - } - } - ] -} +```bash +# Run any existing test suite +# (Check for test directories and scripts in the project) ``` -Now that the debugger knows how to connect to our process in the container/VM and we've set up the necessary source mappings, -go to the "Run and Debug" window and run the "systemd" debug configuration. -If everything goes well, the debugger should now be attached to the systemd instance running in the container/VM. -You can attach breakpoints from the editor and enjoy all the other features of VSCode's debugger. - -To debug systemd components other than PID 1, -set "program" to the full path of the component you want to debug and set "processId" to "${command:pickProcess}". -Now, when starting the debugger, VSCode will ask you the PID of the process you want to debug. -Run `systemctl show --property MainPID --value ` -in the container to figure out the PID and enter it when asked and VSCode will attach to that process instead. +## Code Organization -## Debugging systemd-boot - -During boot, systemd-boot and the stub loader will output messages like `systemd-boot@0x0A` and `systemd-stub@0x0B`, -providing the base of the loaded code. -This location can then be used to attach to a QEMU session (provided it was run with `-s`). -See `debug-sd-boot.sh` script in the tools folder which automates this processes. - -If the debugger is too slow to attach to examine an early boot code passage, -the call to `DEFINE_EFI_MAIN_FUNCTION()` can be modified to enable waiting. -As soon as the debugger has control, we can then run `set variable wait = 0` or `return` to continue. -Once the debugger has attached, setting breakpoints will work like usual. - -To debug systemd-boot in an IDE such as VSCode we can use a launch configuration like this: -```json -{ - "name": "systemd-boot", - "type": "cppdbg", - "request": "launch", - "program": "${workspaceFolder}/build/src/boot/efi/systemd-bootx64.efi", - "cwd": "${workspaceFolder}", - "MIMode": "gdb", - "miDebuggerServerAddress": ":1234", - "setupCommands": [ - { "text": "shell mkfifo /tmp/sdboot.{in,out}" }, - { "text": "shell qemu-system-x86_64 [...] -s -serial pipe:/tmp/sdboot" }, - { "text": "shell ${workspaceFolder}/tools/debug-sd-boot.sh ${workspaceFolder}/build/src/boot/efi/systemd-bootx64.efi /tmp/sdboot.out systemd-boot.gdb" }, - { "text": "source /tmp/systemd-boot.gdb" }, - ] -} -``` +The project is organized with: +- `android/` - Android-specific mobile platform code +- `scripts/` - Build and automation scripts +- `src/` - Core source code +- `docs/` - Documentation -## mkosi + clangd +## Pointer-First Artifact Management -[clangd](https://clangd.llvm.org/) is a language server that provides code completion, diagnostics and more -right in your editor of choice (with the right plugin installed). When using mkosi, we can run clangd in the -mkosi tools tree to avoid needing to install clangd on the host machine. +Large assets should be handled via pointer-first approaches: +- Use `/productenv/src/UserlAss/hf_prepare.sh` for asset fetching +- Never commit large binaries directly to the repository +- Use appropriate secret-gated access for sensitive assets -All that is required is to run `mkosi -f box true` once to make sure the tools tree is available and to modify -the path of the clangd binary used by your editor to the `mkosi.clangd` script included in the systemd repository. -For example, for VScode, you'd have to add the following to the VSCode workspace settings of the systemd repository: +## Getting Help -```json -{ - "clangd.path": "/mkosi/mkosi.clangd", -} -``` - -The script passes any arguments it receives directly to clangd which you can use -for example to tell clangd where the compilation database can be found using the -`--compile-commands-dir=` option. - -When using clangd, it's recommended to setup the build directory containing the -compilation database used by clangd to use clang as the compiler as well: - -```sh -$ mkosi box -- env CC=clang CXX=clang++ meson setup build -``` - -Additionally, the `gensources` target can be used to make sure all generated -sources are generated to avoid clangd complaining that these source files don't -exist. - -```sh -$ mkosi box -- ninja -C build gensources -``` +- File issues on [GitHub Issues](https://github.com/spiralgang/FileSystemds/issues) +- Check existing documentation in this `docs/` directory +- Review the [Mobile Platform README](../README_MOBILE.md) for platform-specific information \ No newline at end of file diff --git a/docs/HOME_DIRECTORY.md b/docs/HOME_DIRECTORY.md deleted file mode 100644 index 2efabaeb928..00000000000 --- a/docs/HOME_DIRECTORY.md +++ /dev/null @@ -1,170 +0,0 @@ ---- -title: Home Directories -category: Users, Groups and Home Directories -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Home Directories - -[`systemd-homed.service(8)`](https://www.freedesktop.org/software/systemd/man/systemd-homed.service.html) -manages home directories of regular ("human") users. -Each directory it manages encapsulates both the data store and the user record of the user, -so that it comprehensively describes the user account, and is thus naturally portable -between systems without any further, external metadata. -This document describes the format used by these home directories, in the context of the storage -mechanism used. - -## General Structure - -Inside of the home directory a file `~/.identity` contains the JSON formatted -user record of the user. -It follows the format defined in [`JSON User Records`](/USER_RECORD). -It is recommended to bring the record into 'normalized' form(i.e. all objects should contain their fields -sorted alphabetically by their key) before storing it there, -though this is not required nor enforced. -Since the user record is cryptographically signed, the user cannot make modifications to the file on their own -(at least not without corrupting it, or knowing the private key used for signing the record). -Note that user records are stored here without their `binding`, `status` and -`secret` sections, i.e. only with the sections included in the signature plus -the signature section itself. - -## Storage Mechanism: Plain Directory/`btrfs` Subvolume - -If the plain directory or `btrfs` subvolume storage mechanism of -`systemd-homed` is used (i.e. `--storage=directory` or `--storage=subvolume` on -the -[`homectl(1)`](https://www.freedesktop.org/software/systemd/man/homectl.html) -command line) the home directory requires no special setup besides including -the user record in the `~/.identity` file. - -It is recommended to name home directories managed this way by -`systemd-homed.service` by the user name, suffixed with `.homedir` -(example: `lennart.homedir` for a user `lennart`) but this is not enforced. -When the user is logged in, the directory is generally mounted to `/home/$USER` -(in our example: `/home/lennart`), thus dropping the suffix while the home directory is active. -`systemd-homed` will automatically discover home directories named this -way in `/home/*.homedir` and synthesize NSS user records for them as they show up. - -## Storage Mechanism: `fscrypt` Directories - -This storage mechanism is mostly identical to the plain directory storage -mechanism, except that the home directory is encrypted using `fscrypt`. -(Use `--storage=fscrypt` on the `homectl` command line.) -Key management is implemented via extended attributes on the directory itself: -for each password an extended attribute `trusted.fscrypt_slot0`, `trusted.fscrypt_slot1`, -`trusted.fscrypt_slot2`, … is maintained. -Its value contains a colon-separated pair of Base64 encoded data fields. -The first field contains a salt value, the second field the encrypted volume key. -The latter is encrypted using AES256 in counter mode, using a key derived from the password via PBKDF2-HMAC-SHA512, -together with the salt value. -The construction is similar to what LUKS does for`dm-crypt` encrypted volumes. -Note that extended attributes are not encrypted by `fscrypt` and hence are suitable for carrying the key slots. -Moreover, by using extended attributes, the slots are directly attached to the directory and -an independent sidecar key database is not required. - -## Storage Mechanism: `cifs` Home Directories - -In this storage mechanism, the home directory is mounted from a CIFS server and -service at login, configured inside the user record. -(Use `--storage=cifs` on the `homectl` command line.) -The local password of the user is used to log into the CIFS service. -The directory share needs to contain the user record in `~/.identity` as well. -Note that this means that the user record needs to be registered locally before it can be mounted for the first time, -since CIFS domain and server information needs to be known *before* the mount. -Note that for all other storage mechanisms it is entirely sufficient if the directories -or storage artifacts are placed at the right locations — all information to -activate them can be derived automatically from their mere availability. - -## Storage Mechanism: `luks` Home Directories - -This is the most advanced and most secure storage mechanism and consists of a -Linux file system inside a LUKS2 volume inside a loopback file (or on removable media). -(Use `--storage=luks` on the `homectl` command line.) Specifically: - -* The image contains a GPT partition table. - For now it should only contain a single partition, - and that partition must have the type UUID - `773f91ef-66d4-49b5-bd83-d683bf40ad16`. - Its partition label must be the user name. - -* This partition must contain a LUKS2 volume, whose label must be the user name. - The LUKS2 volume must contain a LUKS2 token field of type `systemd-homed`. - The JSON data of this token must have a `record` field, containing a string with base64-encoded data. - This data is the JSON user record, in the same serialization as in `~/.identity`, though encrypted. - The JSON data of this token must also have an `iv` field, which contains a - base64-encoded binary initialization vector for the encryption. - The encryption used is the same as the LUKS2 volume itself uses, unlocked by the - same volume key, but based on its own IV. - -* Inside of this LUKS2 volume must be a Linux file system, one of `ext4`, `btrfs` and `xfs`. - The file system label must be the user name. - -* This file system should contain a single directory named after the user. - This directory will become the home directory of the user when activated. - It contains a second copy of the user record in the `~/.identity` file, like in the other storage mechanisms. - -The image file should reside in a directory `/home/` on the system, -named after the user, suffixed with `.home`. -When activated, the container home directory is mounted to the same path, -though with the `.home` suffix dropped — unless a different mount point is defined in the user record. -(e.g.: the loopback file `/home/waldo.home` is mounted to `/home/waldo` while activated.) -When the image is stored on removable media (such as a USB stick), the image -file can be directly `dd`'ed onto it; the format is unchanged. -The GPT envelope should ensure the image is properly recognizable as a home directory both when -used in a loopback file and on a removable USB stick. -(Note that when mounting a home directory from a USB stick, it too defaults to a directory in `/home/`, -named after the username, with no further suffix.) - -Rationale for the GPT partition table envelope: -this way the image is nicely discoverable and recognizable already by partition managers as a home directory. -Moreover, when copied onto a USB stick the GPT envelope makes sure -the stick is properly recognizable as a portable home directory medium. -(Moreover, it allows embedding additional partitions later on, for -example on a multi-purpose USB stick that contains both a home directory and a generic storage volume.) - -Rationale for including the encrypted user record in the LUKS2 header: -Linux kernel file system implementations are generally not robust towards -maliciously formatted file systems; there's a good chance that file system -images can be used as attack vectors, exploiting the kernel. -Thus it is necessary to validate the home directory image *before* mounting it and establishing a minimal level of trust. -Since the user record data is cryptographically signed and user records not signed with a recognized private -key are not accepted, a minimal level of trust between the system and the homedirectory image is established. - -Rationale for storing the home directory one level below to root directory of -the contained file system: -this way special directories such as `lost+found/` do not show up in the user's home directory. - -## Algorithm - -Regardless of the storage mechanism used, an activated home directory -necessarily involves a mount point to be established. -In case of the directory-based storage mechanisms (`directory`, `subvolume` and `fscrypt`) this is a bind mount. -In case of `cifs` this is a CIFS network mount, and in case of the LUKS2 backend a regular block device mount of the file system -contained in the LUKS2 image. -By requiring a mount for all cases (even for those that already are a directory), -a clear logic is defined to distinguish active and inactive home directories, -so that the directories become inaccessible under their regular path the instant they are deactivated. -Moreover, the `nosuid`, `nodev` and `noexec` flags configured in the user record are applied when the bind mount is established. - -During activation, the user records retained on the host, the user record -stored in the LUKS2 header (in case of the LUKS2 storage mechanism) and the -user record stored inside the home directory in `~/.identity` are compared. -Activation is only permitted if they match the same user and are signed by a recognized key. -When the three instances differ in `lastChangeUSec` field, the newest record wins, and is propagated to the other two locations. - -During activation, the file system checker (`fsck`) appropriate for the -selected file system is automatically invoked, ensuring the file system is in a -healthy state before it is mounted. - -If the UID assigned to a user does not match the owner of the home directory in -the file system, the home directory is automatically and recursively `chown()`ed -to the correct UID. - -Depending on the `luksDiscard` setting of the user record, either the backing -loopback file is `fallocate()`ed during activation, or the mounted file system -is `FITRIM`ed after mounting, to ensure the setting is correctly enforced. - -When deactivating a home directory, the file system or block device is trimmed -or extended as configured in the `luksOfflineDiscard` setting of the user -record. diff --git a/docs/INCOMPATIBILITIES.md b/docs/INCOMPATIBILITIES.md deleted file mode 100644 index 784f3a2c026..00000000000 --- a/docs/INCOMPATIBILITIES.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Compatibility with SysV -category: Manuals and Documentation for Users and Administrators -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Compatibility with SysV - -systemd provides a fair degree of compatibility with the behavior exposed by the SysV init system as implemented by many distributions. -Compatibility is provided both for the user experience and the SysV scripting APIs. -However, there are some areas where compatibility is limited due to technical reasons or design decisions of systemd and the distributions. -All of the following applies to SysV init scripts handled by systemd, however a number of them matter only on specific distributions. -Many of the incompatibilities are specific to distribution-specific extensions of LSB/SysV init. - -* If your distribution removes SysV init scripts in favor of systemd unit files typing "/etc/init.d/foobar start" to start a service will not work, since the script will not be available. Use the more correct "/sbin/service foobar start" instead, and your command will be forwarded to systemd. Note that invoking the init script directly has always been suboptimal since too much of the caller's execution context (environment block, umask, resource limits, audit trails, ...) ended up being inherited by the service, and invocation via "/sbin/service" used to clean this up at least partially. Invocation via /sbin/service works on both SysV and systemd systems. Also, LSB only standardizes invocation via "/sbin/service" anyway. (Note that some distributions ship both systemd unit files and SysV scripts for the services. For these invoking the init scripts will work as expected and the request be forwarded to systemd in any case.) -* LSB header dependency information matters. The SysV implementations on many distributions did not use the dependency information encoded in LSB init script headers, or used them only in very limited ways. Due to that they are often incorrect or incomplete. systemd however fully interprets these headers and follows them closely at runtime (and not at installation time like some implementations). -* Timeouts apply to all init script operations in systemd. While on SysV systems a hanging init script could freeze the system on systemd all init script operations are subject to a timeout of 5min. -* Services are executed in completely clean execution contexts, no context of the invoking user session is inherited. Not even $HOME or similar are set. Init scripts depending on these will not work correctly. -* Services cannot read from stdin, as this will be connected to /dev/null. That means interactive init scripts are not supported (i.e. Debian's X-Interactive in the LSB header is not supported either.) Thankfully most distributions do not support interaction in init scripts anyway. If you need interaction to ask disk or SSL passphrases please consider using the minimal password querying framework systemd supports. ([details](/PASSWORD_AGENTS), [manual page](http://0pointer.de/public/systemd-man/systemd-ask-password.html)) -* Additional verbs for init scripts are not supported. If your init script traditionally supported additional verbs for your init script simply move them to an auxiliary script. -* Additional parameters to the standard verbs (i.e. to "start", "stop" and "status") are not supported. This was an extension of SysV that never was standardized officially, and is not supported in systemd. -* Overriding the "restart" verb is not supported. This verb is always implemented by systemd itself, and consists of a "stop" followed by a "start". -* systemd only stops running services. On traditional SysV a K link installed for shutdown was executed when going down regardless whether the service was started before or not. systemd is more strict here and does not stop service that weren't started in the first place. -* Note that neither S nor K links for runlevels 0 and 6 have any effect. Running services will be terminated anyway when shutting down, and no new SysV services are started at shut down. -* If systemd doesn't know which PID is the main PID of a service, it will not be able to track its runtime, and hence a service exiting on its own will not make systemd consider it stopped. Use the Red Hat "pidfile:" syntax in the SysV script header comment block to let systemd know which PID file (and hence PID) belongs to your service. Note that systemd cannot know if a SysV service is one of the kind where the runtime is defined by a specific process or whether it is one where there is none, hence the requirement of explicit configuration of a PID file in order to make systemd track the process lifetime. (Note that the Red Hat "pidfile:" stanza may only appear once in init scripts.) -* Runlevels are supported in a limited fashion only. SysV runlevels are mapped to systemd target units, however not all systemd target units map back to SysV runlevels. This is due to the fact that systemd targets are a lot more flexible and expressive than SysV runlevels. That means that checks for the current runlevel (with /sbin/runlevel or so) may well return "N" (i.e. unknown runlevel) during normal operation. Scripts that rely on explicit runlevel checks are incompatible with many setups. Avoid runlevel checks like these. -* Tools like /sbin/chkconfig might return misleading information when used to list enablement status of services. First of all, the tool will only see SysV services, not native units. Secondly, it will only show runlevel-related information (which does not fully map to systemd targets). Finally, the information shown might be overridden by a native unit file. -* By default runlevels 2,3,4 are all aliases for "multi-user.target". If a service is enabled in one of these runlevels, they'll be enabled in all of these. This is only a default however, and users can easily override the mappings, and split them up into individual runlevels if they want. However, we recommend moving on from runlevels and using the much more expressive target units of systemd. -* Early boot runlevels as they are used by some distributions are no longer supported. i.e. "fake", distribution-specific runlevels such as "S" or "b" cannot be used with systemd. -* On SysV systems changes to init scripts or any other files that define the boot process (such as /etc/fstab) usually had an immediate effect on everything started later. This is different on systemd-based systems where init script information and other boot-time configuration files are only reread when "systemctl daemon-reload" is issued. (Note that some commands, notably "systemctl enable"/"systemctl disable" do this implicitly however.) This is by design, and a safety feature, since it ensures that half-completed changes are not read at the wrong time. -* Multiple entries for the same mount path in /etc/fstab are not supported. In systemd there's only a single unit definition for each mount path read at any time. Also the listing order of mounts in /etc/fstab has no effect, mounts are executed in parallel and dependencies between them generated automatically depending on path prefixes and source paths. -* systemd's handling of the existing "nofail" mount option in /etc/fstab is stricter than it used to be on some sysvinit distributions: mount points that fail and are not listed as "nofail" will cause the boot to be stopped, for security reasons, as we should not permit unprivileged code to run without everything listed — and not expressly exempted through "nofail" — being around. Hence, please mark all mounts where booting shall proceed regardless whether they succeeded or not with "nofail" -* Some SysV systems support an "rc.local" script that is supposed to be called "last" during boot. In systemd, the script is supported, but the semantics are less strict, as there is simply no concept of "last service", as the boot process is event- and request-based, parallelized and compositive. In general, it's a good idea to write proper unit files with properly defined dependencies, and avoid making use of rc.local. -* systemd assumes that the UID boundary between system and regular users is a choice the distribution makes, and not the administrator. Hence it expects this setting as compile-time option to be picked by the distribution. It will _not_ check /etc/login.defs during runtime. - -Note that there are some areas where systemd currently provides a certain amount of compatibility where we expect this compatibility to be removed eventually. diff --git a/docs/INITRD_INTERFACE.md b/docs/INITRD_INTERFACE.md deleted file mode 100644 index 402b6a93b7a..00000000000 --- a/docs/INITRD_INTERFACE.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Initrd Interface -category: Interfaces -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - - -# The initrd Interface of systemd - -The Linux initrd mechanism (short for "initial RAM disk", also known as -"initramfs") refers to a small file system archive that is unpacked by the -kernel and contains the first userspace code that runs. It typically finds and -transitions into the actual root file system to use. systemd supports both -initrd and initrd-less boots. If an initrd is used, it is a good idea to pass a -few bits of runtime information from the initrd to systemd in order to avoid -duplicate work and to provide performance data to the administrator. In this -page we attempt to roughly describe the interfaces that exist between the -initrd and systemd. These interfaces are currently used by -[mkosi](https://github.com/systemd/mkosi)-generated initrds, dracut and the -Arch Linux initrds. - -* The initrd should mount `/run/` as a tmpfs and pass it pre-mounted when - jumping into the main system when executing systemd. The mount options should - be `mode=0755,nodev,nosuid,strictatime`. - -* It's highly recommended that the initrd also mounts `/usr/` (if split off) as - appropriate and passes it pre-mounted to the main system, to avoid the - problems described in [Booting without /usr is Broken](/SEPARATE_USR_IS_BROKEN). - -* If the executable `/run/initramfs/shutdown` exists systemd will use it to - jump back into the initrd on shutdown. `/run/initramfs/` should be a usable - initrd environment to which systemd will pivot back and the `shutdown` - executable in it should be able to detach all complex storage that for - example was needed to mount the root file system. It's the job of the initrd - to set up this directory and executable in the right way so that this works - correctly. The shutdown binary is invoked with the shutdown verb as `argv[1]`, - optionally followed (in `argv[2]`, `argv[3]`, …) systemd's original command - line options, for example `--log-level=` and similar. - -* Storage daemons run from the initrd should follow the guide on - [systemd and Storage Daemons for the Root File System](/ROOT_STORAGE_DAEMONS) - to survive properly from the boot initrd all the way to the point where - systemd jumps back into the initrd for shutdown. - -One last clarification: we use the term _initrd_ very generically here -describing any kind of early boot file system, regardless whether that might be -implemented as an actual ramdisk, ramfs or tmpfs. We recommend using _initrd_ -in this sense as a term that is unrelated to the actual backing technologies -used. - -## Using systemd inside an initrd - -It is also possible and recommended to implement the initrd itself based on -systemd. Here are a few terse notes: - -* Provide `/etc/initrd-release` in the initrd image. The idea is that it - follows the same format as the usual `/etc/os-release` but describes the - initrd implementation rather than the OS. systemd uses the existence of this - file as a flag whether to run in initrd mode, or not. - -* When run in initrd mode, systemd and its components will read a couple of - additional command line arguments, which are generally prefixed with `rd.` - -* To transition into the main system image invoke `systemctl switch-root`. - -* The switch-root operation will result in a killing spree of all running - processes. Some processes might need to be excluded from that, see the guide - on [systemd and Storage Daemons for the Root File System](/ROOT_STORAGE_DAEMONS). diff --git a/docs/MOUNT_REQUIREMENTS.md b/docs/MOUNT_REQUIREMENTS.md deleted file mode 100644 index 7f61b41ff4f..00000000000 --- a/docs/MOUNT_REQUIREMENTS.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -title: Mount Requirements -category: Booting -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Mount Point Availability Requirements - -systemd makes various requirements on the time during boot where various parts -of the Linux file system hierarchy must be available and must be mounted. If -the file systems backing these mounts are located on external or remote media, -that require special drivers, infrastructure or networking to be set up, then -this implies that this functionality must be started and running at that point -already. - -Generally, there are three categories of requirements: - -1. 🌥️ *initrd*: File system mounts that must be established before the OS - transitions into the root file system. (i.e. that must be stablished from - the initrd before the initrd→host transition takes place.) - -2. 🌤️ *early*: File system mounts that must be established during early boot, - after the initrd→host transition took place, but before regular services are - started. (i.e. before `local-fs.target` is reached.) - -3. ☀️ *regular*: File system mounts that can be mounted at any time during the - boot process – but which specific, individual services might require to be - established at the point they are started. (i.e. these mounts are typically - ordered before `remote-fs.target`.) - -Of course, mounts that fall into category 3 can also be mounted during the -initrd or in early boot. And those from category 2 can also be mounted already -from the initrd. - -Here's a table with relevant mounts and to which category they belong: - -| *Mount* | *Category* | -|---------------|------------| -| `/` (root fs) | 1 | -| `/usr/` | 1 | -| `/etc/` | 1 | -| `/var/` | 2 | -| `/var/tmp/` | 2 | -| `/tmp/` | 2 | -| `/home/` | 3 | -| `/srv/` | 3 | -| XBOOTLDR | 3 | -| ESP | 3 | - -Or in other words: the root file system (obviously…), `/usr/` and `/etc/` (if -these are split off) must be mounted at the moment the initrd transitions into -the host. Then, `/var/` (with `/var/tmp/`) and `/tmp/` (if split off) must be -mounted, before the host reaches `local-fs.target` (and then `basic.target`), -after which any remaining mounts may be established. - -If mounts such as `/var/` are not mounted during early boot (or from the -initrd), and require some late boot service (for example a network manager -implementation) to operate this will likely result in cyclic ordering -dependencies, and will result in various forms of boot failures. - -Also note that the whole of `/var/` (including `/var/tmp/`), and `/tmp/` must -be *writable* at the moment indicated above. It's OK if they are mounted -read-only at an earlier time as long as they are remounted writable by the -indicated point in time. Systems where these three hierarchies remain read-only -during regular operation are not supported by `systemd`. (Note that for -stateless systems it is absolutely OK and supported to mount an empty `tmpfs` -there at boot, `systemd` will know how to populate the tree as necessary.) - -If you intend to use network-backed mounts (NFS, SMB, iSCSI, NVME-TCP and -similar, including anything you add the `_netdev` pseudo mount option to) for -any of the mounts from category 1 or 2, make sure to use a network managing -implementation that is capable of running from the initrd/during early -boot. [`systemd-networkd(8)`](https://www.freedesktop.org/software/systemd/man/latest/systemd-networkd.html) -for example works well in such scenarios. - -Note that -[`systemd-homed.service(8)`](https://www.freedesktop.org/software/systemd/man/latest/systemd-homed.html) -(which is a regular service, i.e. runs after `basic.target`) requires `/home/` -to be mounted. diff --git a/docs/PAX_CONTROL_GROUPS.md b/docs/PAX_CONTROL_GROUPS.md deleted file mode 100644 index 44919592d34..00000000000 --- a/docs/PAX_CONTROL_GROUPS.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -title: Pax Controla Groupiana -category: Users, Groups and Home Directories -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Pax Controla Groupiana - -_aka "How to behave nicely in the cgroupfs trees"_ - -**Important Update: Please consult this document only as a historical reference. -It was written under the assumption that the cgroups tree was a shared resource. -However, after much discussion this concept has been deemed outdated. -The cgroups tree can no longer be considered a shared resource. -Instead, a management daemon of some kind needs to arbitrate access to it, and it needs to actively propagate changes between the entities it manages. -More specifically, on systemd systems this management daemon is systemd itself, accessible via a number of bus APIs. -This means instead of dealing directly with the low-level interfaces of the cgroup file system, please use systemd's high-level APIs as a replacement, see the -[New Control Group Interfaces](/CONTROL_GROUP_INTERFACE) -for details. They offer similar functionality.** - -Are you writing an application interfacing with the cgroups tree? -The cgroups trees are a shared resource, other applications will use them too. -Here are a few recommendations how to write your application in a way that minimizes conflicts with other applications. -If you follow these guidelines applications should not step on any other application's toes and users will be happy. - -Before you read these recommendations please make sure you understand cgroups thoroughly, -and specifically are aware what a controller is, what a named hierarchy is and so on. - -## Intended Audience - -You should consider these recommendations if you are you working on one of the following: - -- You write a system or session manager based on cgroups (like systemd) -- You write a VM manager based on cgroups (like libvirt) -- You write a terminal application and want to place every shell in a separate cgroup (like gnome-terminal) -- You write a web browser and want to place every renderer in a separate cgroup (like Firefox or Chrome) -- You create a container for some purpose (such as systemd-nspawn) -- Or you use cgroups for any other purpose and want things to work nicely with other applications. - -## General Recommendations - -- If you use one of the kernel controllers, do _not_ assume you are the only one who uses them. - Other programs may manipulate the tree, add cgroups and change group attributes at any time, and they will not inform you about it. - The kernel provided controller hierarchies are a shared resource, so be nice. -- If you use a generic named hierarchy with no controller attached, then you may assume it's yours and only yours, and that no other programs interfere with it. -- If you use a generic named hierarchy with no controller attached, then make sure to name it after your project in order to minimize namespacing conflicts. - A hierarchy named "name=web" is a bit generic. - A hierarchy named "name=apache" a much better choice, if you are an Apache developer and need an entire hierarchy all for yourself. -- Do _not_ assume everybody uses the same library to manipulate the cgroups tree as you are. - In fact most likely most applications and the user himself will manipulate the tree without any further indirection (i.e. will use naked system calls/shell commands) -- Never create cgroups at the top of the tree (i.e. with an absolute path). - If possible find the cgroup your own process was started in and create subgroups only below that group (read /proc/self/cgroup to find it). - If that's not applicable, then at least place yourself below the cgroup path of PID 1 (read /proc/1/cgroup to find it). - This is important to ensure that containers work properly (the cgroupfs tree is currently not virtualized for containers!), and solves permission problems, and makes the whole system nicely stackable. -- A corollary of this: If you spawn subprocesses expect that they will create subcgroups. - That means when terminating there might be subcgroups below the ones you created and you hence need to recursively remove them too. - In fact, many of your operations must probably be executed in a recursive fashion. -- Do not play permission games: if you are an unprivileged user application then it's _not_ your business to ensure you have the right permissions - (i.e. do not include any setuid code in your app to create groups). - Instead your system manager (such as systemd), - should provide you with the right set of permissions on the cgroup you are running in to create subgroups. - Normally that should mean that depending on administrator configuration, you will or will not get access to create subgroups under the cgroup you are running in and the ability to add PIDs to it. - If you don't get access to these hierarchies then this might be a decision by the administrator and you should do your best to go on, and fail gracefully. -- If you create a cgroup, then you are in charge of removing it too after using it. - Do not remove other program's cgroups. - Special exception: in some cases it is OK to pre-set attributes on certain cgroups that are primarily managed by another program. - (Example: in systemd we are fine if you externally pre-create or manipulate service cgroups, for example to make changes to some attributes you cannot control with systemd natively, see below). - In that case: create the cgroup and set the sticky bit (+t) on the tasks file in it. - This will then be used as an indication to the primary manager of the group not to remove the cgroup at the end, in order to avoid that your settings are lost. - This is of course a bit of a misuse of the sticky bit, but given that it serves no other purpose on Linux for normal files, it is an OK use, with a fitting meaning given the name of "sticky bit". -- If you find a process in a cgroup you are about to remove, and it is not yours, consider leaving the cgroup around. - I.e. if rmdir returns EEMPTY, ignore this. -- The cgroup mount point for a specific hierarchy is /sys/fs/cgroup/$CONTROLLER/. - (Example: /sys/fs/cgroup/cpu for the "cpu" controller). - In your application you are welcome to rely on these standardized mount points, - and it is not necessary to dynamically determine the current mount point via /proc/self/mountinfo (but if you do, that's of course fine, too). - Note that /sys/fs/cgroup/$CONTROLLER/ might actually just be a symlink to some other mount point (see below). -- If multiple controllers are mounted into the same hierarchy, it is guaranteed that symlinks exist to make sure all jointly mounted controllers are still available under /sys/fs/cgroup/$CONTROLLER/. - Example: if "cpu" and "cpuacct" are mounted together, then symlinks /sys/fs/cgroup/cpu and /sys/fs/cgroup/cpuacct will point to the joint mountpoint (which could be something like /sys/fs/cgroup/cpu+cpuacct). -- Your application should not mount the cgroup controller file systems (unless it is your own private named hierarchy). - This is exclusively a job for the system manager or a system-wide init script such as cgconfig. - If you work on a system manager or such an init script you must mount the cgroup controllers to /sys/fs/cgroup/$CONTROLLER/ or provide compatibility symlinks. -- It's a good idea not to fail if a cgroup already exists when you try to create it. - Ignore EEXIST on mkdir. -- Avoid renaming cgroups or similar fancier file operations. -- Expect that other programs might readjust the attributes on your cgroups dynamically during runtime. -- When creating a cgroup pick a nice a descriptive name that is guessable and no surprise to the admin. - The admin will thank you for this if he has to read the output of "ps -eo pid,args,cgroups" -- /sys/fs/cgroup is a tmpfs. If you create your own private named hierarchy then you are welcome to mount it into a subdirectory of this directory. - This minimizes surprises for the user. -- /sys/fs/cgroup is a tmpfs, but it's only intended use is to act as place where control group hierarchies can be mounted or symlinked to. - You should not place any other kind of file in this directory. - The same way as /dev/shm is for POSIX shared memory segments only -- and nothing else -- this directory is for cgroup hierarchies only. - Just because something is a tmpfs it doesn't mean you can actually use it for "temporary" files, thank you. -- Avoid creating orthogonal hierarchies in the various kernel controller hierarchies. - Please make sure that the controllers contain the same hierarchy or subsets of each other. - -## Cooperation with systemd - -systemd adheres to the recommendations above and guarantees additional behavior which might be useful for writing applications that cooperate with systemd on cgroup management: - -- If a service cgroup already exists, systemd will make use of it and not recreate it. - (If +t is set on the tasks file it will not remove it when stopping a service, otherwise it will, see above). - It is hence OK to pre-create cgroups and then let systemd use it, without having systemd remove it afterwards. -- If a service cgroup already exists, systemd will not override the attributes of the cgroup with the exception of those explicitly configured in the systemd unit files. - It is hence OK to pre-create cgroups for use in systemd, and pre-apply attributes to it. -- To avoid that systemd places all services in automatic cgroups in the "cpu" hierarchy change the DefaultControllers= in /etc/systemd/system.conf and set it to the empty string. -- By default systemd will place services only in automatic cgroups in the "cpu" hierarchy and in its own private tree "name=systemd". - If you want it to duplicate these trees in other hierarchies add them to DefaultControllers= in /etc/systemd/system.conf -- To opt-out or opt-in specific services from the automatic tree generation in the kernel controller hierarchies use ControlGroup= in the unit file. - Use "ControlGroup=cpu:/" to opt-out of cgroup assignment for a service or "ControlGroup=cpu:/foo/bar" to manipulate the cgroup path. -- Stay away from the name=systemd named hierarchy. - It's private property of systemd. - You are welcome to explore it, but it is uncool to modify it from outside systemd. -Thanks. diff --git a/docs/PORTABILITY_AND_STABILITY.md b/docs/PORTABILITY_AND_STABILITY.md deleted file mode 100644 index 42d86f278e0..00000000000 --- a/docs/PORTABILITY_AND_STABILITY.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Interface Portability and Stability -category: Interfaces -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Interface Portability and Stability Promise - -systemd provides various interfaces developers and programs might rely on. -Starting with version 26 (the first version released with Fedora 15) we promise to keep a number of them stable and compatible for the future. - -The stable interfaces are: - -* **The unit configuration file format**. Unit files written now will stay compatible with future versions of systemd. - Extensions to the file format will happen in a way that existing files remain compatible. - -* **The command line interface** of `systemd`, `systemctl`, `loginctl`, `journalctl`, and all other command line utilities installed in `$PATH` and documented in a man page. - We will make sure that scripts invoking these commands will continue to work with future versions of systemd. - Note however that the output generated by these commands is generally not included in the promise, unless it is documented in the man page. - Example: the output of `systemctl status` is not stable, but that of `systemctl show` is, because the former is intended to be human-readable and the latter computer readable, and this is documented in the man page. - -* **The protocol spoken on the socket referred to by `$NOTIFY_SOCKET`**, as documented in - [sd_notify(3)](https://www.freedesktop.org/software/systemd/man/sd_notify.html). Note that, although using - libsystemd is a good choice, this protocol can also be reimplemented without external dependencies, as - demonstrated in the example listed in - [sd_notify(3)](https://www.freedesktop.org/software/systemd/man/devel/sd_notify.html#Notes) - -* Some of the **"special" unit names** and their semantics. - To be precise the ones that are necessary for normal services, and not those required only for early boot and late shutdown, with very few exceptions. - To list them here: `basic.target`, `shutdown.target`, `sockets.target`, `network.target`, `getty.target`, `graphical.target`, `multi-user.target`, `rescue.target`, `emergency.target`, `poweroff.target`, `reboot.target`, `halt.target`, `runlevel[1-5].target`. - -* **The D-Bus interfaces of the main service daemon and other daemons**. We try to always preserve backwards compatibility, and intentional breakage is never introduced. - Nevertheless, when we find bugs that mean that the existing interface was not useful, or when the implementation did something different than stated by the documentation and the implemented behaviour is not useful, we will fix the implementation and thus introduce a change in behaviour. - But the API (parameter counts and types) is never changed, and existing attributes and methods will not be removed. - -* For a more comprehensive and authoritative list, consult the chart below. - -The following interfaces will not necessarily be kept stable for now, but we will eventually make a stability promise for these interfaces too. -In the meantime we will however try to keep breakage of these interfaces at a minimum: - -* **The set of states of the various state machines used in systemd**, e.g. the high-level unit states inactive, active, deactivating, and so on, as well (and in particular) the low-level per-unit states. - -* **All "special" units that aren't listed above**. - -The following interfaces are considered private to systemd, and are not and will not be covered by any stability promise: - -* **Undocumented switches** to `systemd`, `systemctl` and otherwise. - -* **The internal protocols** used on the various sockets such as the sockets `/run/systemd/shutdown`, `/run/systemd/private`. - -One of the main goals of systemd is to unify basic Linux configurations and service behaviors across all distributions. -Systemd project does not contain any distribution-specific parts. -Distributions are expected to convert over time their individual configurations to the systemd format, or they will need to carry and maintain patches in their package if they still decide to stay different. - -What does this mean for you? When developing with systemd, don't use any of the latter interfaces, or we will tell your mom, and she won't love you anymore. -You are welcome to use the other interfaces listed here, but if you use any of the second kind (i.e. those where we don't yet make a stability promise), then make sure to subscribe to our mailing list, where we will announce API changes, and be prepared to update your program eventually. - -Note that this is a promise, not an eternal guarantee. -These are our intentions, but if in the future there are very good reasons to change or get rid of an interface we have listed above as stable, then we might take the liberty to do so, despite this promise. -However, if we do this, then we'll do our best to provide a smooth and reasonably long transition phase. - - -## Interface Portability And Stability Chart - -systemd provides a number of APIs to applications. -Below you'll find a table detailing which APIs are considered stable and how portable they are. - -This list is intended to be useful for distribution and OS developers who are interested in maintaining a certain level of compatibility with the new interfaces systemd introduced, without relying on systemd itself. - -In general it is our intention to cooperate through interfaces and not code with other distributions and OSes. -That means that the interfaces where this applies are best reimplemented in a compatible fashion on those other operating systems. -To make this easy we provide detailed interface documentation where necessary. -That said, it's all Open Source, hence you have the option to a) fork our code and maintain portable versions of the parts you are interested in independently for your OS, or b) build systemd for your distro, but leave out all components except the ones you are interested in and run them without the core of systemd involved. -We will try not to make this any more difficult than necessary. -Patches to allow systemd code to be more portable will be accepted on case-by-case basis (essentially, patches to follow well-established standards instead of e.g. glibc or linux extensions have a very high chance of being accepted, while patches which make the code ugly or exist solely to work around bugs in other projects have a low chance of being accepted). - -Many of these interfaces are already being used by applications and 3rd party code. -If you are interested in compatibility with these applications, please consider supporting these interfaces in your distribution, where possible. - - -## General Portability of systemd and its Components - -**Portability to OSes:** systemd is not portable to non-Linux systems. -It makes use of a large number of Linux-specific interfaces, including many that are used by its very core. -We do not consider it feasible to port systemd to other Unixes (let alone non-Unix operating systems) and will not accept patches for systemd core implementing any such portability (but hey, it's git, so it's as easy as it can get to maintain your own fork...). -APIs that are supposed to be used as library code are exempted from this: it is important to us that these compile nicely on non-Linux and even non-Unix platforms, even if they might just become NOPs. - -**Portability to Architectures:** It is important to us that systemd is portable to little endian as well as big endian systems. -We will make sure to provide portability with all important architectures and hardware Linux runs on and are happy to accept patches for this. - -**Portability to Distributions:** It is important to us that systemd is portable to all Linux distributions. -However, the goal is to unify many of the needless differences between the distributions, and hence will not accept patches for certain distribution-specific work-arounds. -Compatibility with the distribution's legacy should be maintained in the distribution's packaging, and not in the systemd source tree. - -**Compatibility with Specific Versions of Other packages:** We generally avoid adding compatibility kludges to systemd that work around bugs in certain versions of other software systemd interfaces with. We strongly encourage fixing bugs where they are, and if that's not systemd we rather not try to fix it there. -(There are very few exceptions to this rule possible, and you need an exceptionally strong case for it). - - -## General Portability of systemd's APIs - -systemd's APIs are available everywhere where systemd is available. -Some of the APIs we have defined are supposed to be generic enough to be implementable independently of systemd, thus allowing compatibility with systems systemd itself is not compatible with, i.e. other OSes, and distributions that are unwilling to fully adopt systemd. - -A number of systemd's APIs expose Linux or systemd-specific features that cannot sensibly be implemented elsewhere. -Please consult the table below for information about which ones these are. - -Note that not all of these interfaces are our invention (but most), we just adopted them in systemd to make them more prominently implemented. -For example, we adopted many Debian facilities in systemd to push it into the other distributions as well. - - ---- - - -And now, here's the list of (hopefully) all APIs that we have introduced with systemd: - -| API | Type | Covered by Interface Stability Promise | Fully documented | Known External Consumers | Reimplementable Independently | Known Other Implementations | systemd Implementation portable to other OSes or non-systemd distributions | -| --- | ---- | ----------------------------------------------------------------------------------------- | ---------------- | ------------------------ | ----------------------------- | --------------------------- | -------------------------------------------------------------------------- | -| [hostnamed](https://www.freedesktop.org/software/systemd/man/org.freedesktop.hostname1.html) | D-Bus | yes | yes | GNOME | yes | [Ubuntu](https://launchpad.net/ubuntu/+source/ubuntu-system-service), [Gentoo](http://www.gentoo.org/proj/en/desktop/gnome/openrc-settingsd.xml), [BSD](http://uglyman.kremlin.cc/gitweb/gitweb.cgi?p=systembsd.git;a=summary) | partially | -| [localed](https://www.freedesktop.org/software/systemd/man/org.freedesktop.locale1.html) | D-Bus | yes | yes | GNOME | yes | [Ubuntu](https://launchpad.net/ubuntu/+source/ubuntu-system-service), [Gentoo](http://www.gentoo.org/proj/en/desktop/gnome/openrc-settingsd.xml), [BSD](http://uglyman.kremlin.cc/gitweb/gitweb.cgi?p=systembsd.git;a=summary) | partially | -| [timedated](https://www.freedesktop.org/software/systemd/man/org.freedesktop.timedate1.html) | D-Bus | yes | yes | GNOME | yes | [Gentoo](http://www.gentoo.org/proj/en/desktop/gnome/openrc-settingsd.xml), [BSD](http://uglyman.kremlin.cc/gitweb/gitweb.cgi?p=systembsd.git;a=summary) | partially | -| [initrd interface](/INITRD_INTERFACE) | Environment, flag files | yes | yes | mkosi, dracut, ArchLinux | yes | ArchLinux | no | -| [Container interface](/CONTAINER_INTERFACE) | Environment, Mounts | yes | yes | libvirt/LXC | yes | - | no | -| [Boot Loader interface](/BOOT_LOADER_INTERFACE) | EFI variables | yes | yes | gummiboot | yes | - | no | -| [Service bus API](https://www.freedesktop.org/software/systemd/man/org.freedesktop.systemd1.html) | D-Bus | yes | yes | system-config-services | no | - | no | -| [logind](https://www.freedesktop.org/software/systemd/man/org.freedesktop.login1.html) | D-Bus | yes | yes | GNOME | no | - | no | -| [sd-bus.h API](https://www.freedesktop.org/software/systemd/man/sd-bus.html) | C Library | yes | yes | - | maybe | - | maybe | -| [sd-daemon.h API](https://www.freedesktop.org/software/systemd/man/sd-daemon.html) | C Library or Drop-in | yes | yes | numerous | yes | - | yes | -| [sd-device.h API](https://www.freedesktop.org/software/systemd/man/sd-device.html) | C Library | yes | no | numerous | yes | - | yes | -| [sd-event.h API](https://www.freedesktop.org/software/systemd/man/sd-event.html) | C Library | yes | yes | - | maybe | - | maybe | -| [sd-gpt.h API](https://www.freedesktop.org/software/systemd/man/sd-gpt.html) | Header Library | yes | no | - | yes | - | yes | -| [sd-hwdb.h API](https://www.freedesktop.org/software/systemd/man/sd-hwdb.html) | C Library | yes | yes | - | maybe | - | yes | -| [sd-id128.h API](https://www.freedesktop.org/software/systemd/man/sd-id128.html) | C Library | yes | yes | - | yes | - | yes | -| [sd-journal.h API](https://www.freedesktop.org/software/systemd/man/sd-journal.html) | C Library | yes | yes | - | maybe | - | no | -| [sd-login.h API](https://www.freedesktop.org/software/systemd/man/sd-login.html) | C Library | yes | yes | GNOME, polkit, ... | no | - | no | -| [sd-messages.h API](https://www.freedesktop.org/software/systemd/man/sd-messages.html) | Header Library | yes | yes | - | yes | python-systemd | yes | -| [sd-path.h API](https://www.freedesktop.org/software/systemd/man/sd-path.html) | C Library | yes | no | - | maybe | - | maybe | -| [$XDG_RUNTIME_DIR](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) | Environment | yes | yes | glib, GNOME | yes | - | no | -| [$LISTEN_FDS $LISTEN_PID FD Passing](https://www.freedesktop.org/software/systemd/man/sd_listen_fds.html) | Environment | yes | yes | numerous (via sd-daemon.h) | yes | - | no | -| [$NOTIFY_SOCKET Daemon Notifications](https://www.freedesktop.org/software/systemd/man/sd_notify.html) | Environment | yes | yes | a few, including udev | yes | - | no | -| [argv[0][0]='@' Logic](/ROOT_STORAGE_DAEMONS) | `/proc` marking | yes | yes | mdadm | yes | - | no | -| [Unit file format](https://www.freedesktop.org/software/systemd/man/systemd.unit.html) | File format | yes | yes | numerous | no | - | no | -| [Network](https://www.freedesktop.org/software/systemd/man/systemd.network.html) & [Netdev file format](https://www.freedesktop.org/software/systemd/man/systemd.netdev.html) | File format | yes | yes | no | no | - | no | -| [Link file format](https://www.freedesktop.org/software/systemd/man/systemd.link.html) | File format | yes | yes | no | no | - | no | -| [Journal File Format](/JOURNAL_FILE_FORMAT) | File format | yes | yes | - | maybe | - | no | -| [Journal Export Format](JOURNAL_EXPORT_FORMATS#journal-export-format) | File format | yes | yes | - | yes | - | no | -| [Journal JSON Format](JOURNAL_EXPORT_FORMATS#journal-json-format) | File format | yes | yes | - | yes | - | no | -| [Password Agents](/PASSWORD_AGENTS) | Socket+Files | yes | yes | - | yes | - | no | -| [udev multi-seat properties](https://www.freedesktop.org/software/systemd/man/sd-login.html) | udev Property | yes | yes | X11, gdm | no | - | no | -| udev session switch ACL properties | udev Property | no | no | - | no | - | no | -| [CLI of systemctl,...](https://www.freedesktop.org/software/systemd/man/systemctl.html) | CLI | yes | yes | numerous | no | - | no | -| [tmpfiles.d](https://www.freedesktop.org/software/systemd/man/tmpfiles.d.html) | File format | yes | yes | numerous | yes | ArchLinux | partially | -| [sysusers.d](https://www.freedesktop.org/software/systemd/man/sysusers.d.html) | File format | yes | yes | unknown | yes | | partially | -| [/etc/machine-id](https://www.freedesktop.org/software/systemd/man/machine-id.html) | File format | yes | yes | D-Bus | yes | - | no | -| [binfmt.d](https://www.freedesktop.org/software/systemd/man/binfmt.d.html) | File format | yes | yes | numerous | yes | - | partially | -| [/etc/hostname](https://www.freedesktop.org/software/systemd/man/hostname.html) | File format | yes | yes | numerous (it's a Debian thing) | yes | Debian, ArchLinux | no | -| [/etc/locale.conf](https://www.freedesktop.org/software/systemd/man/locale.conf.html) | File format | yes | yes | - | yes | ArchLinux | partially | -| [/etc/machine-info](https://www.freedesktop.org/software/systemd/man/machine-info.html) | File format | yes | yes | - | yes | - | partially | -| [modules-load.d](https://www.freedesktop.org/software/systemd/man/modules-load.d.html) | File format | yes | yes | numerous | yes | - | partially | -| [/usr/lib/os-release](https://www.freedesktop.org/software/systemd/man/os-release.html) | File format | yes | yes | some | yes | Fedora, OpenSUSE, ArchLinux, Angstrom, Frugalware, others... | no | -| [sysctl.d](https://www.freedesktop.org/software/systemd/man/sysctl.d.html) | File format | yes | yes | some (it's a Debian thing) | yes | procps/Debian, ArchLinux | partially | -| [/etc/timezone](https://www.freedesktop.org/software/systemd/man/timezone.html) | File format | yes | yes | numerous (it's a Debian thing) | yes | Debian | partially | -| [/etc/vconsole.conf](https://www.freedesktop.org/software/systemd/man/vconsole.conf.html) | File format | yes | yes | - | yes | ArchLinux | partially | -| `/run` | File hierarchy change | yes | yes | numerous | yes | OpenSUSE, Debian, ArchLinux | no | -| [Generators](https://www.freedesktop.org/software/systemd/man/systemd.generator.html) | Subprocess | yes | yes | - | no | - | no | -| [System Updates](https://www.freedesktop.org/software/systemd/man/systemd.offline-updates.html) | System Mode | yes | yes | - | no | - | no | -| [Presets](https://www.freedesktop.org/software/systemd/man/systemd.preset.html) | File format | yes | yes | - | no | - | no | -| Udev rules | File format | yes | yes | numerous | no | no | partially | - - -### Explanations - -Items for which "systemd implementation portable to other OSes" is "partially" means that it is possible to run the respective tools that are included in the systemd tarball outside of systemd. -Note however that this is not officially supported, so you are more or less on your own if you do this. -If you are opting for this solution simply build systemd as you normally would but drop all files except those which you are interested in. - -Of course, it is our intention to eventually document all interfaces we defined. -If we haven't documented them for now, this is usually because we want the flexibility to still change things, or don't want 3rd party applications to make use of these interfaces already. -That said, our sources are quite readable and open source, so feel free to spelunk around in the sources if you want to know more. - -If you decide to reimplement one of the APIs for which "Reimplementable independently" is "no", then we won't stop you, but you are on your own. - -This is not an attempt to comprehensively list all users of these APIs. -We are just listing the most obvious/prominent ones which come to our mind. - -Of course, one last thing I can't make myself not ask you before we finish here, and before you start reimplementing these APIs in your distribution: -are you sure it's time well spent if you work on reimplementing all this code instead of just spending it on adopting systemd on your distro as well? - -## Independent Operation of systemd Programs - -Some programs in the systemd suite are intended to operate independently of the -running init process (or even without an init process, for example when -creating system installation chroots). They can be safely called on systems with -a different init process or for example in package installation scriptlets. - -The following programs currently and in the future will support operation -without communicating with the `systemd` process: -`systemd-escape`, -`systemd-id128`, -`systemd-path`, -`systemd-tmpfiles`, -`systemd-sysctl`, -`systemd-sysusers`. - -Many other programs support operation without the system manager except when -the specific functionality requires such communication. For example, -`journalctl` operates almost independently, but will query the boot id when -`--boot` option is used; it also requires `systemd-journald` (and thus -`systemd`) to be running for options like `--flush` and `--sync`. -`systemd-journal-remote`, `systemd-journal-upload`, `systemd-journal-gatewayd`, -`coredumpctl`, `busctl`, `systemctl --root` also fall into this category of -mostly-independent programs. diff --git a/docs/PORTABLE_SERVICES.md b/docs/PORTABLE_SERVICES.md deleted file mode 100644 index 1766865fe0f..00000000000 --- a/docs/PORTABLE_SERVICES.md +++ /dev/null @@ -1,367 +0,0 @@ ---- -title: Portable Services Introduction -category: Concepts -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Portable Services - -systemd (since version 239) supports a concept of "Portable Services". -"Portable Services" are a delivery method for system services that uses -two specific features of container management: - -1. Applications are bundled. I.e. multiple services, their binaries and all - their dependencies are packaged in an image, and are run directly from it. - -2. Stricter default security policies, i.e. sand-boxing of applications. - -The primary tool for interacting with Portable Services is `portablectl`, -and they are managed by the `systemd-portabled` service. - -Portable services don't bring anything inherently new to the table. -All they do is put together known concepts to cover a specific set of use-cases in a -slightly nicer way. - -## So, what *is* a "Portable Service"? - -A portable service is ultimately just an OS tree, either inside of a directory, -or inside a raw disk image containing a Linux file system. -This tree is called the "image". It can be "attached" or "detached" from the system. -When "attached", specific systemd units from the image are made available on the -host system, then behaving pretty much exactly like locally installed system services. -When "detached", these units are removed again from the host, leaving -no artifacts around (except maybe messages they might have logged). - -The OS tree/image can be created with any tool of your choice. -For example, you can use `dnf --installroot=` if you like, or `debootstrap`, the image format is -entirely generic, and doesn't have to carry any specific metadata beyond what distribution images carry anyway. -Or to say this differently: -The image format doesn't define any new metadata as unit files and OS tree directories or disk -images are already sufficient, and pretty universally available these days. -One particularly nice tool for creating suitable images is -[mkosi](https://github.com/systemd/mkosi), -but many other existing tools will do too. - -Portable services may also be constructed from layers, similarly to container environments. -See [Extension Images](#extension-images) below. - -If you so will, "Portable Services" are a nicer way to manage `chroot()` -environments, with better security, tooling and behavior. - -## Where's the difference to a "Container"? - -"Container" is a very vague term, after all it is used for -systemd-nspawn/LXC-type OS containers, for Docker/rkt-like micro service -containers, and even certain 'lightweight' VM runtimes. - -"Portable services" do not provide a fully isolated environment to the payload, like containers mostly intend to. -Instead, they are more like regular system services, can be controlled with the same tools, are exposed the same way in all infrastructure, and so on. -The main difference is that they use a different root directory than the rest of the system. -Hence, the intent is not to run code in a different, isolated environment from the host — like most containers would — but to run it in the same environment, but with stricter access controls on what the service can see and do. - -One point of differentiation: since programs running as "portable services" are -pretty much regular system services, they won't run as PID 1 (like they would -under Docker), but as normal processes. - -A corollary of that is that they aren't supposed to manage anything in their own environment (such as the network) as the execution environment is mostly shared with the rest of the system. - -The primary focus use-case of "portable services" is to extend the host system -with encapsulated extensions, but provide almost full integration with the rest -of the system, though possibly restricted by security knobs. -This focus includes system extensions otherwise sometimes called "super-privileged containers". - -Note that portable services are only available for system services, not for -user services (i.e. the functionality cannot be used for the stuff -[`bubblewrap`](https://github.com/containers/bubblewrap)/[`flatpak`](https://flatpak.org/) is focusing on). - -## Mode of Operation - -If you have a portable service image, maybe in a raw disk image called -`foobar_0.7.23.raw`, then attaching the services to the host is as easy as: - -``` -# portablectl attach foobar_0.7.23.raw -``` - -This command does the following: - -1. It dissects the image, checks and validates the `os-release` file of the - image, and looks for all included unit files. - -2. It copies out all unit files with a suffix of `.service`, `.socket`, - `.target`, `.timer` and `.path`, whose name begins with the image's name - (with `.raw` removed), truncated at the first underscore if there is one. - This prefix name generated from the image name must be followed by a ".", - "-" or "@" character in the unit name. Or in other words, given the image - name of `foobar_0.7.23.raw` all unit files matching - `foobar-*.{service|socket|target|timer|path}`, - `foobar@.{service|socket|target|timer|path}` as well as - `foobar.*.{service|socket|target|timer|path}` and - `foobar.{service|socket|target|timer|path}` - are copied out. - These unit files are placed in `/etc/systemd/system.attached/` - (which is part of the normal unit file search path of PID 1, and thus loaded exactly like regular unit - files). - Within the images the unit files are looked for at the usual locations, i.e. in `/usr/lib/systemd/system/` and `/etc/systemd/system/` and so on, relative to the image's root. - -3. For each such unit file a drop-in file is created. - Let's say `foobar-waldo.service` was one of the unit files copied to - `/etc/systemd/system.attached/`, then a drop-in file - `/etc/systemd/system.attached/foobar-waldo.service.d/20-portable.conf` is - created, containing a few lines of additional configuration: - - ``` - [Service] - RootImage=/path/to/foobar.raw - Environment=PORTABLE=foobar - LogExtraFields=PORTABLE=foobar - ``` - -4. For each such unit a "profile" drop-in is linked in. - This "profile" drop-in generally contains security options that lock down the service. - By default the `default` profile is used, which provides a medium level of security. - There's also `trusted`, which runs the service with no restrictions, i.e. in - the host file system root and with full privileges. - The `strict` profile comes with the toughest security restrictions. - Finally, `nonetwork` is like `default` but without network access. - Users may define their own profiles too (or modify the existing ones). - -And that's already it. - -Note that the images need to stay around (and in the same location) as long as the -portable service is attached. -If an image is moved, the `RootImage=` line written to the unit drop-in would point to an non-existent path, and break access to the image. - -The `portablectl detach` command executes the reverse operation: -it looks for the drop-ins and the unit files associated with the image, and removes them. - -Note that `portablectl attach` won't enable or start any of the units it copies -out by default, but `--enable` and `--now` parameter are available as shortcuts. -The same is true for the opposite `detach` operation. - -The `portablectl reattach` command combines a `detach` with an `attach`. -It is useful in case an image gets upgraded, as it allows performing a `restart` -operation on the units instead of `stop` plus `start`, thus providing lower -downtime and avoiding losing runtime state associated with the unit such as the -file descriptor store. - -## Requirements on Images - -Note that portable services don't introduce any new image format, but most OS -images should just work the way they are. -Specifically, the following requirements are made for an image that can be attached/detached with `portablectl`. - -1. It must contain an executable that shall be invoked, along with all its - dependencies. - Any binary code needs to be compiled for an architecture compatible with the host. - -2. The image must either be a plain sub-directory (or btrfs subvolume) - containing the binaries and its dependencies in a classic Linux OS tree, or - must be a raw disk image either containing only one, naked file system, or - an image with a partition table understood by the Linux kernel with only a - single partition defined, or alternatively, a GPT partition table with a set - of properly marked partitions following the - [Discoverable Partitions Specification](https://uapi-group.org/specifications/specs/discoverable_partitions_specification). - -3. The image must at least contain one matching unit file, with the right name - prefix and suffix (see above). - The unit file is searched in the usual paths, i.e. primarily `/etc/systemd/system/` and `/usr/lib/systemd/system/` within the image. - (The implementation will check a couple of other paths too, but it's recommended to use these two paths.) - -4. The image must contain an os-release file, either in `/etc/os-release` or - `/usr/lib/os-release`. The file should follow the standard format. - -5. The image must contain the files `/etc/resolv.conf` and `/etc/machine-id` - (empty files are ok), they will be bind mounted from the host at runtime. - -6. The image must contain directories `/proc/`, `/sys/`, `/dev/`, `/run/`, - `/tmp/`, `/var/tmp/` that can be mounted over with the corresponding version - from the host. - -7. The OS might require other files or directories to be in place. - For example, if the image is built based on glibc, the dynamic loader needs to be - available in `/lib/ld-linux.so.2` or `/lib64/ld-linux-x86-64.so.2` (or - similar, depending on architecture), and if the distribution implements a - merged `/usr/` tree, this means `/lib` and/or `/lib64` need to be symlinks - to their respective counterparts below `/usr/`. - For details see your distribution's documentation. - -Note that images created by tools such as `debootstrap`, `dnf --installroot=` -or `mkosi` generally satisfy all of the above. -If you wonder what the most minimal image would be that complies with the requirements above, it could -consist of this: - -``` -/usr/bin/minimald # a statically compiled binary -/usr/lib/systemd/system/minimal-test.service # the unit file for the service, with ExecStart=/usr/bin/minimald -/usr/lib/os-release # an os-release file explaining what this is -/etc/resolv.conf # empty file to mount over with host's version -/etc/machine-id # ditto -/proc/ # empty directory to use as mount point for host's API fs -/sys/ # ditto -/dev/ # ditto -/run/ # ditto -/tmp/ # ditto -/var/tmp/ # ditto -``` - -And that's it. - -Note that qualifying images do not have to contain an init system of their own. -If they do, it's fine, it will be ignored by the portable service logic, -but they generally don't have to, and it might make sense to avoid any, to keep images minimal. - -If the image is writable, and some of the files or directories that are -overmounted from the host do not exist yet they will be automatically created. -On read-only, immutable images (e.g. `erofs` or `squashfs` images) all files -and directories to over-mount must exist already. - -Note that as no new image format or metadata is defined, it's very -straightforward to define images than can be made use of in a number of different ways. -For example, by using `mkosi -b` you can trivially build a -single, unified image that: - -1. Can be attached as portable service, to run any container services natively - on the host. - -2. Can be run as OS container, using `systemd-nspawn`, by booting the image - with `systemd-nspawn -i -b`. - -3. Can be booted directly as VM image, using a generic VM executor such as - `virtualbox`/`qemu`/`kvm` - -4. Can be booted directly on bare-metal systems. - -Of course, to facilitate 2, 3 and 4 you need to include an init system in the image. -To facilitate 3 and 4 you also need to include a boot loader in the -image. -As mentioned, `mkosi -b` takes care of all of that for you, but any other image generator should work too. - -The -[os-release(5)](https://www.freedesktop.org/software/systemd/man/os-release.html) -file may optionally be extended with a `PORTABLE_PREFIXES=` field listing all -supported portable service prefixes for the image (see above). -This is useful for informational purposes (as it allows recognizing portable service images -from their contents as such), but is also useful to protect the image from -being used under a wrong name and prefix. -This is particularly relevant if the images are cryptographically authenticated (via Verity or a similar mechanism) as this way the (not necessarily authenticated) image file name can be -validated against the (authenticated) image contents. -If the field is not specified the image will work fine, but is not necessarily recognizable as -portable service image, and any set of units included in the image may be attached, there are no restrictions enforced. - -## Extension Images - -Portable services can be delivered as one or multiple images that extend the base -image, and are combined with OverlayFS at runtime, when they are attached. -This enables a workflow that splits the base 'runtime' from the daemon, so that multiple -portable services can share the same 'runtime' image (libraries, tools) without -having to include everything each time, with the layering happening only at runtime. -The `--extension` parameter of `portablectl` can be used to specify as many upper -layers as desired. -On top of the requirements listed in the previous section, the following must also be observed: - -1. The base/OS image must contain an `os-release file`, either in `/etc/os-release` - or `/usr/lib/os-release`, in the standard format. - -2. The upper extension images must contain an extension-release file in - `/usr/lib/extension-release.d/`, with an `ID=` and `SYSEXT_LEVEL=`/`VERSION_ID=` - matching the base image for sysexts, or `/etc/extension-release.d/`, with an - `ID=` and `CONFEXT_LEVEL=`/`VERSION_ID=` matching the base image for confexts. - -3. The base/OS image does not need to have any unit files. - -4. The upper sysext images must contain at least one matching unit file each, - with the right name prefix and suffix (see above). Confext images do not have - to contain units. - -5. As with the base/OS image, each upper extension image must be a plain - sub-directory, btrfs subvolume, or a raw disk image. - -``` -# portablectl attach --extension foobar_0.7.23.raw debian-runtime_11.1.raw foobar -# portablectl attach --extension barbaz_7.0.23/ debian-runtime_11.1.raw barbaz -``` - -## Execution Environment - -Note that the code in portable service images is run exactly like regular services. -Hence there's no new execution environment to consider. -And, unlike Docker would do it, as these are regular system services they aren't run as PID -1 either, but with regular PID values. - -## Access to host resources - -If services shipped with this mechanism shall be able to access host resources -(such as files or AF_UNIX sockets for IPC), use the normal `BindPaths=` and -`BindReadOnlyPaths=` settings in unit files to mount them in. -In fact, the `default` profile mentioned above makes use of this to ensure -`/etc/resolv.conf`, the D-Bus system bus socket or write access to the logging -subsystem are available to the service. - -## Instantiation - -Sometimes it makes sense to instantiate the same set of services multiple times. -The portable service concept does not introduce a new logic for this. -It is recommended to use the regular systemd unit templating for this, i.e. to -include template units such as `foobar@.service`, so that instantiation is as -simple as: - -``` -# portablectl attach foobar_0.7.23.raw -# systemctl enable --now foobar@instancea.service -# systemctl enable --now foobar@instanceb.service -… -``` - -The benefit of this approach is that templating works exactly the same for -units shipped with the OS itself as for attached portable services. - -## Immutable images with local data - -It's a good idea to keep portable service images read-only during normal operation. -In fact, all but the `trusted` profile will default to this kind of behaviour, by setting the `ProtectSystem=strict` option. -In this case writable service data may be placed on the host file system. -Use `StateDirectory=` in the unit files to enable such behaviour and add a local data directory to the -services copied onto the host. - -## Logging - -Several fields are autotmatically added to log messages generated by a portable -service (or about a portable service, e.g.: start/stop logs from systemd). -The `PORTABLE=` field will refer to the name of the portable image where the unit -was loaded from. In case extensions are used, additionally there will be a `PORTABLE_ROOT=` field, referring to the name of the image used as the base layer (i.e.: `RootImage=` or `RootDirectory=`), and one `PORTABLE_EXTENSION=` field for -each extension image used. - -The `os-release` file from the portable image will be parsed and added as structured metadata to the journal log entries. -The parsed fields will be the first ID field which is set from the set of `IMAGE_ID` and `ID` in this order of preference, and the first version field which is set from a set of `IMAGE_VERSION`, `VERSION_ID`, and `BUILD_ID` in this order of preference. -The ID and version, if any, are concatenated with an underscore (`_`) as separator. -If only either one is found, it will be used by itself. -The field will be named `PORTABLE_NAME_AND_VERSION=`. - -In case extensions are used, the same fields in the same order, but prefixed by -`SYSEXT_`/`CONFEXT_`, are parsed from each `extension-release` file, and are appended -to the journal as log entries, using `PORTABLE_EXTENSION_NAME_AND_VERSION=` as the field name. -The base layer's field will be named `PORTABLE_ROOT_NAME_AND_VERSION=` instead of `PORTABLE_NAME_AND_VERSION=` in this case. - -For example, a portable service `app0` using two extensions `app0.raw` and -`app1.raw` (with `SYSEXT_ID=app`, and `SYSEXT_VERSION_ID=` `0` and `1` in their -respective extension-releases), and a base layer `base.raw` (with `VERSION_ID=10` and -`ID=debian` in `os-release`), will create log entries with the following fields: - -``` -PORTABLE=app0.raw -PORTABLE_ROOT=base.raw -PORTABLE_ROOT_NAME_AND_VERSION=debian_10 -PORTABLE_EXTENSION=app0.raw -PORTABLE_EXTENSION_NAME_AND_VERSION=app_0 -PORTABLE_EXTENSION=app1.raw -PORTABLE_EXTENSION_NAME_AND_VERSION=app_1 -``` - -## Links - -[`portablectl(1)`](https://www.freedesktop.org/software/systemd/man/portablectl.html)
-[`systemd-portabled.service(8)`](https://www.freedesktop.org/software/systemd/man/systemd-portabled.service.html)
-[Walkthrough for Portable Services](https://0pointer.net/blog/walkthrough-for-portable-services.html)
-[Repo with examples](https://github.com/systemd/portable-walkthrough) diff --git a/docs/PORTING_TO_NEW_ARCHITECTURES.md b/docs/PORTING_TO_NEW_ARCHITECTURES.md deleted file mode 100644 index a4dc6c29dd5..00000000000 --- a/docs/PORTING_TO_NEW_ARCHITECTURES.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: Porting to New Architectures -category: Contributing -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Porting systemd to New Architectures - -Here's a brief checklist of things to implement when porting systemd to a new -architecture. - -1. Patch - [src/basic/architecture.h](https://github.com/systemd/systemd/blob/main/src/basic/architecture.h) - and - [src/basic/architecture.c](https://github.com/systemd/systemd/blob/main/src/basic/architecture.c) - to make your architecture known to systemd. Besides an `ARCHITECTURE_XYZ` - enumeration entry you need to provide an implementation of - `native_architecture()` and `uname_architecture()`. - -2. Patch - [src/shared/gpt.h](https://github.com/systemd/systemd/blob/main/src/shared/gpt.h) - and - [src/shared/gpt.c](https://github.com/systemd/systemd/blob/main/src/shared/gpt.c) - and define a new set of GPT partition type UUIDs for the root file system, - `/usr/` file system, and the matching Verity and Verity signature - partitions. Use `systemd-id128 new -p` to generate new suitable UUIDs you - can use for this. Make sure to register your new types in the various - functions in `gpt.c`. Also make sure to update the tables in - [Discoverable Partitions Specification](https://uapi-group.org/specifications/specs/discoverable_partitions_specification) - and `man/systemd-gpt-auto-generator.xml` accordingly. - -3. If your architecture supports UEFI, make sure to update the `efi_arch` - variable logic in `meson.build` to be set to the right architecture string - as defined by the UEFI specification. (This ensures that `systemd-boot` will - be built as the appropriately named `BOOT.EFI` binary.) Also, if your - architecture uses a special boot protocol for the Linux kernel, make sure to - implement it in `src/boot/efi/linux*.c`, so that the `systemd-stub` EFI stub - can work. - -4. Make sure to register the right system call numbers for your architecture in - `src/basic/missing_syscall_def.h`. systemd uses various system calls the - Linux kernel provides that are currently not wrapped by glibc (or are only - in very new glibc), and we need to know the right numbers for them. It might - also be necessary to tweak `src/basic/raw-clone.h`. - -5. Make sure the code in `src/shared/seccomp-util.c` properly understands the - local architecture and its system call quirks. - -6. If your architecture uses a `/lib64/` library directory, then make sure that - the `BaseFilesystem` table in `src/shared/base-filesystem.c` has an entry - for it so that it can be set up automatically if missing. This is useful to - support booting into OS trees that have an empty root directory with only - `/usr/` mounted in. - -7. If your architecture supports VM virtualization and provides CPU opcodes - similar to x86' CPUID, consider adding native support for detecting VMs this - way to `src/basic/virt.c`. diff --git a/docs/PREDICTABLE_INTERFACE_NAMES.md b/docs/PREDICTABLE_INTERFACE_NAMES.md deleted file mode 100644 index 9fa9fea7304..00000000000 --- a/docs/PREDICTABLE_INTERFACE_NAMES.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -title: Predictable Network Interface Names -category: Networking -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Predictable Network Interface Names - -Starting with v197 systemd/udev will automatically assign predictable, stable network interface names for all local Ethernet, WLAN and WWAN interfaces. -This is a departure from the traditional interface naming scheme (`eth0`, `eth1`, `wlan0`, ...), but should fix real problems. - -## Why? - -The classic naming scheme for network interfaces applied by the kernel is to simply assign names beginning with `eth0`, `eth1`, ... to all interfaces as they are probed by the drivers. -As the driver probing is generally not predictable for modern technology this means that as soon as multiple network interfaces are available the assignment of the names `eth0`, `eth1` and so on is generally not fixed anymore and it might very well happen that `eth0` on one boot ends up being `eth1` on the next. -This can have serious security implications, for example in firewall rules which are coded for certain naming schemes, and which are hence very sensitive to unpredictable changing names. - -To fix this problem multiple solutions have been proposed and implemented. -For a longer time udev shipped support for assigning permanent `ethX` names to certain interfaces based on their MAC addresses. -This turned out to have a multitude of problems, among them: this required a writable root directory which is generally not available; the statelessness of the system is lost as booting an OS image on a system will result in changed configuration of the image; on many systems MAC addresses are not actually fixed, such as on a lot of embedded hardware and particularly on all kinds of virtualization solutions. -The biggest of all however is that the userspace components trying to assign the interface name raced against the kernel assigning new names from the same `ethX` namespace, a race condition with all kinds of weird effects, among them that assignment of names sometimes failed. -As a result support for this has been removed from systemd/udev a while back. - -Another solution that has been implemented is `biosdevname` which tries to find fixed slot topology information in certain firmware interfaces and uses them to assign fixed names to interfaces which incorporate their physical location on the mainboard. -In a way this naming scheme is similar to what is already done natively in udev for various device nodes via `/dev/*/by-path/` symlinks. -In many cases, biosdevname departs from the low-level kernel device identification schemes that udev generally uses for these symlinks, and instead invents its own enumeration schemes. - -Finally, many distributions support renaming interfaces to user-chosen names (think: `internet0`, `dmz0`, ...) keyed off their MAC addresses or physical locations as part of their networking scripts. -This is a very good choice but does have the problem that it implies that the user is willing and capable of choosing and assigning these names. - -We believe it is a good default choice to generalize the scheme pioneered by `biosdevname`. -Assigning fixed names based on firmware/topology/location information has the big advantage that the names are fully automatic, fully predictable, that they stay fixed even if hardware is added or removed (i.e. no reenumeration takes place) and that broken hardware can be replaced seamlessly. -That said, they admittedly are sometimes harder to read than the `eth0` or `wlan0` everybody is used to. Example: `enp5s0` - - -## What precisely has changed in v197? - -With systemd 197 we have added native support for a number of different naming policies into systemd/udevd proper and made a scheme similar to biosdevname's (but generally more powerful, and closer to kernel-internal device identification schemes) the default. -The following different naming schemes for network interfaces are now supported by udev natively: - -1. Names incorporating Firmware/BIOS provided index numbers for on-board devices (example: `eno1`) -1. Names incorporating Firmware/BIOS provided PCI Express hotplug slot index numbers (example: `ens1`) -1. Names incorporating physical/geographical location of the connector of the hardware (example: `enp2s0`) -1. Names incorporating the interfaces's MAC address (example: `enx78e7d1ea46da`) -1. Classic, unpredictable kernel-native ethX naming (example: `eth0`) - -By default, systemd v197 will now name interfaces following policy 1) if that information from the firmware is applicable and available, falling back to 2) if that information from the firmware is applicable and available, falling back to 3) if applicable, falling back to 5) in all other cases. -Policy 4) is not used by default, but is available if the user chooses so. - -This combined policy is only applied as last resort. -That means, if the system has biosdevname installed, it will take precedence. -If the user has added udev rules which change the name of the kernel devices these will take precedence too. -Also, any distribution specific naming schemes generally take precedence. - - -## Come again, what good does this do? - -With this new scheme you now get: - -* Stable interface names across reboots -* Stable interface names even when hardware is added or removed, i.e. no re-enumeration takes place (to the level the firmware permits this) -* Stable interface names when kernels or drivers are updated/changed -* Stable interface names even if you have to replace broken ethernet cards by new ones -* The names are automatically determined without user configuration, they just work -* The interface names are fully predictable, i.e. just by looking at lspci you can figure out what the interface is going to be called -* Fully stateless operation, changing the hardware configuration will not result in changes in `/etc` -* Compatibility with read-only root -* The network interface naming now follows more closely the scheme used for aliasing block device nodes and other device nodes in `/dev` via symlinks -* Applicability to both x86 and non-x86 machines -* The same on all distributions that adopted systemd/udev -* It's easy to opt out of the scheme (see below) - -Does this have any drawbacks? Yes, it does. -Previously it was practically guaranteed that hosts equipped with a single ethernet card only had a single `eth0` interface. -With this new scheme in place, an administrator now has to check first what the local interface name is before they can invoke commands on it, where previously they had a good chance that `eth0` was the right name. - - -## I don't like this, how do I disable this? - -You basically have three options: - -1. You disable the assignment of fixed names, so that the unpredictable kernel names are used again. For this, simply mask udev's .link file for the default policy: `ln -s /dev/null /etc/systemd/network/99-default.link` -1. You create your own manual naming scheme, for example by naming your interfaces `internet0`, `dmz0` or `lan0`. For that create your own `.link` files in `/etc/systemd/network/`, that choose an explicit name or a better naming scheme for one, some, or all of your interfaces. See [systemd.link(5)](https://www.freedesktop.org/software/systemd/man/systemd.link.html) for more information. -1. You pass the `net.ifnames=0` on the kernel command line - -## How does the new naming scheme look like, precisely? - -That's documented in detail the [systemd.net-naming-scheme(7)](https://www.freedesktop.org/software/systemd/man/systemd.net-naming-scheme.html) man page. -Please refer to this in case you are wondering how to decode the new interface names. diff --git a/docs/RELEASE.md b/docs/RELEASE.md deleted file mode 100644 index aefe430e3ec..00000000000 --- a/docs/RELEASE.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: Steps to a Successful Release -category: Contributing -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Steps to a Successful Release - -1. Add all items to NEWS -2. Update the contributors list in NEWS (`ninja -C build git-contrib`) -3. Update the time and place in NEWS -4. Update hwdb (`ninja -C build update-hwdb`, `ninja -C build update-hwdb-autosuspend`, commit separately). -5. Update syscall numbers (`ninja -C build update-syscall-tables update-syscall-header`). -6. [RC1] Update library numbers in `meson.build` -7. Update version number in `meson.version` (e.g. from `256~devel` to `256~rc1` or from `256~rc3` to `256`). Note that this uses a tilde (\~) instead of a hyphen (-) because tildes sort lower in version comparisons according to the [version format specification](https://uapi-group.org/specifications/specs/version_format_specification/), and we want `255~rc1` to sort lower than `255`. -8. Check dbus docs with `ninja -C build update-dbus-docs` -9. Check manpages list with `ninja -C build update-man-rules` -10. Update translation strings (`ninja -C build systemd-pot`, `ninja -C build systemd-update-po`) - drop the header comments from `systemd.pot` + re-add SPDX before committing. If the only change in a file is the 'POT-Creation-Date' field, then ignore that file. -11. Tag the release: `version="v$(sed 's/~/-/g' meson.version)" && git tag -s "${version}" -m "systemd ${version}"` (tildes are replaced with hyphens, because git doesn't accept the former). -12. Do `ninja -C build` -13. Make sure that the version string and package string match: `build/systemctl --version` -14. [FINAL] Close the github milestone and open a new one (https://github.com/systemd/systemd/milestones) -15. "Draft" a new release on github (https://github.com/systemd/systemd/releases/new), mark "This is a pre-release" if appropriate. -16. Check that announcement to systemd-devel, with a copy&paste from NEWS, was sent. This should happen automatically. -17. Update IRC topic (`/msg chanserv TOPIC #systemd Version NNN released | Online resources https://systemd.io/`) -18. [FINAL] Create an empty -stable branch: `git push systemd origin/main:refs/heads/v${version}-stable`. -19. [FINAL] Build and upload the documentation (on the -stable branch): `ninja -C build doc-sync` -20. [FINAL] Change the Github Pages branch to the newly created branch (https://github.com/systemd/systemd/settings/pages) and set the 'Custom domain' to 'systemd.io' -21. [FINAL] Update version number in `meson.version` to the devel version of the next release (e.g. from `256` to `257~devel`) - -# Steps to a Successful Stable Release - -1. Backport at least the commits from all PRs tagged with `needs-stable-backport` on Github with `git cherry-pick -x`. Any other commits that fix bugs, change documentation, tests, CI or mkosi can generally be backported as well. Since 256 the stable branches live [here](https://github.com/systemd/systemd/). Stable branches for older releases are available [here](https://github.com/systemd/systemd-stable/). Check each commit to see if it makes sense to backport and check the comments on the PR to see if the author indicated that only specific commits should be backported. -2. Update the version number in `meson.version` (e.g. from `256.2` to `256.3`) (only for 256-stable or newer) -3. Tag the release: `version="v$(cat meson.version)" && git tag -s "${version}" -m "systemd-stable ${version}"` (Fill in the version manually on releases older than 256) diff --git a/docs/RESOLVED-VPNS.md b/docs/RESOLVED-VPNS.md deleted file mode 100644 index dbf43f9a384..00000000000 --- a/docs/RESOLVED-VPNS.md +++ /dev/null @@ -1,268 +0,0 @@ ---- -title: systemd-resolved and VPNs -category: Networking -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# `systemd-resolved.service` and VPNs - -`systemd-resolved.service` supports routing lookups for specific domains to specific -interfaces. This is useful for hooking up VPN software with systemd-resolved -and making sure the exact right lookups end up on the VPN and on the other -interfaces. - -For a verbose explanation of `systemd-resolved.service`'s domain routing logic, -see its [man -page](https://www.freedesktop.org/software/systemd/man/systemd-resolved.service.html). This -document is supposed to provide examples to use the concepts for the specific -purpose of managing VPN DNS configuration. - -Let's first define two distinct VPN use-cases: - -1. *Corporate* VPNs, i.e. VPNs that open access to a specific set of additional - hosts. Only specific domains should be resolved via the VPN's DNS servers, - and everything that is not related to the company's domain names should go - to regular, non-VPN DNS instead. - -2. *Privacy* VPNs, i.e. VPNs that should be used for basically all DNS traffic, - once they are up. If this type of VPN is used, any regular, non-VPN DNS - servers should not get any traffic anymore. - -Then, let's briefly introduce three DNS routing concepts that software managing -a network interface may configure. - -1. Search domains: these are traditional DNS configuration parameters and are - used to suffix non-qualified domain names (i.e. single-label ones), to turn - them into fully qualified domain names. Traditionally (before - `systemd-resolved.service`), search domain names are attached to a system's - IP configuration as a whole, in `systemd-resolved.service` they are - associated to individual interfaces instead, since they are typically - acquired through some network associated concept, such as a DHCP, IPv6RA or - PPP lease. Most importantly though: in `systemd-resolved.service` they are - not just used to suffix single-label domain names, but also for routing - domain name lookups: if a network interface has a search domain `foo.com` - configured on it, then any lookups for names ending in `.foo.com` (or for - `foo.com` itself) are preferably routed to the DNS servers configured on the - same network interface. - -2. Routing domains: these are very similar to search domains, but are purely - about DNS domain name lookup routing — they are not used for qualifying - single-label domain names. When it comes to routing, assigning a routing - domain to a network interface is identical to assigning a search domain to - it. - - Why the need to have both concepts, i.e. search *and* routing domains? - Mostly because in many cases the qualifying of single-label names is not - desirable (as it has security implications), but needs to be supported for - specific use-cases. Routing domains are a concept `systemd-resolved.service` - introduced, while search domains are traditionally available and are part of - DHCP/IPv6RA/PPP leases and thus universally supported. In many cases routing - domains are probably the more appropriate concept, but not easily available, - since they are not part of DHCP/IPv6RA/PPP. - - Routing domains for `systemd-resolved.service` are usually presented along - with search domains in mostly the same way, but prefixed with `~` to - differentiate them. i.e. `~foo.com` is a configured routing domain, while - `foo.com` would be a configured search domain. - - One routing domain is particularly interesting: `~.` — the catch-all routing - domain. (The *dot* domain `.` is how DNS denotes the "root" domain, i.e. the - parent domain of all domains, but itself.) When used on an interface any DNS - traffic is preferably routed to its DNS servers. (A search domain – i.e. `.` - instead of `~.` — would have the same effect, but given that it's mostly - pointless to suffix an unqualified domain with `.`, we generally declare it - as a routing domain, not a search domain). - - Routing domains also have particular relevance when it comes to the reverse - lookup DNS domains `.in-addr.arpa` and `.ip6.arpa`. An interface that has - these (or sub-domains thereof) defined as routing domains, will be preferably - used for doing reverse IP to domain name lookups. e.g. declaring - `~168.192.in-addr.arpa` on an interface means that all lookups to find the - domain names for IPv4 addresses 192.168.x.y are preferably routed to it. - -3. The `default-route` boolean. This is a simple boolean value that may be set - on an interface. If true (the default), any DNS lookups for which no - matching routing or search domains are defined are routed to interfaces - marked like this. If false then the DNS servers on this interface are not - considered for routing lookups to except for the ones listed in the - search/routing domain list. An interface that has no search/routing domain - associated and also has this boolean off is not considered for *any* - lookups. - -One more thing to mention: in `systemd-resolved.service` if lookups match the -search/routing domains of multiple interfaces at once, then they are sent to -all of them in parallel, and the first positive reply used. If all lookups fail -the last negative reply is used. This means the DNS zones on the relevant -interfaces are "merged": domains existing on one but not the other will "just -work" and vice versa. - -And one more note: the domain routing logic implemented is a tiny bit more -complex that what described above: if there two interfaces have search domains -that are suffix of each other, and a name is looked up that matches both, the -interface with the longer match will win and get the lookup routed to is DNS -servers. Only if the match has the same length, then both will be used in -parallel. Example: one interface has `~foo.example.com` as routing domain, and -another one `example.com` has search domain. A lookup for -`waldo.foo.example.com` is the exclusively routed to the first interface's DNS -server, since it matches by three suffix labels instead of just two. The fact -that the matching length is taken into consideration for the routing decision -is particularly relevant if you have one interface with the `~.` routing domain -and another one with `~corp.company.example` — both suffixes match a lookup for -`foo.corp.company.example`, but the latter interface wins, since the match is -for four labels, while the other is for zero labels. - -## Putting it Together - -Let's discuss how the three DNS routing concepts above are best used for a -reasonably complex scenario consisting of: - -1. One VPN interface of the *corporate* kind, maybe called `company0`. It makes - available a bunch of servers, all in the domain `corp.company.example`. - -2. One VPN interface of the *privacy* kind, maybe called `privacy0`. When it is - up all DNS traffic shall preferably routed to its DNS servers. - -3. One regular WiFi interface, maybe called `wifi0`. It has a regular DNS - server on it. - -Here's how to best configure this for `systemd-resolved.service`: - -1. `company0` should get a routing domain `~corp.company.example` - configured. (A search domain `corp.company.example` would work too, if - qualifying of single-label names is desired or the VPN lease information - does not provide for the concept of routing domains, but does support search - domains.) This interface should also set `default-route` to false, to ensure - that really only the DNS lookups for the company's servers are routed there - and nothing else. Finally, it might make sense to also configure a routing - domain `~2.0.192.in-addr.arpa` on the interface, ensuring that all IPv4 - addresses from the 192.0.2.x range are preferably resolved via the DNS - server on this interface (assuming that that's the IPv4 address range the - company uses internally). - -2. `privacy0` should get a routing domain `~.` configured. The setting of - `default-route` for this interface is then irrelevant. This means: once the - interface is up, all DNS traffic is preferably routed there. - -3. `wifi0` should not get any special settings, except possibly whatever the - local WiFi router considers suitable as search domain, for example - `fritz.box`. The default `true` setting for `default-route` is good too. - -With this configuration if only `wifi0` is up, all DNS traffic goes to its DNS -server, since there are no other interfaces with better matching DNS -configuration. If `privacy0` is then upped, all DNS traffic will exclusively go -to this interface now — with the exception of names below the `fritz.box` -domain, which will continue to go directly to `wifi0`, as the search domain -there says so. Now, if `company0` is also upped, it will receive DNS traffic -for the company's internal domain and internal IP subnet range, but nothing -else. If `privacy0` is then downed again, `wifi0` will get the regular DNS -traffic again, and `company0` will still get the company's internal domain and -IP subnet traffic and nothing else. Everything hence works as intended. - -## How to Implement this in Your VPN Software - -Most likely you want to expose a boolean in some way that declares whether a -specific VPN is of the *corporate* or the *privacy* kind: - -1. If managing a *corporate* VPN, you configure any search domains the user or - the VPN contact point provided. And you set `default-route` to false. If you - have IP subnet information for the VPN, it might make sense to insert - `~….in-addr.arpa` and `~….ip6.arpa` reverse lookup routing domains for it. - -2. If managing a *privacy* VPN, you include `~.` in the routing domains, the - value for `default-route` is actually irrelevant, but I'd set it to true. No - need to configure any reverse lookup routing domains for it. - -(If you also manage regular WiFi/Ethernet devices, just configure them as -traditional, i.e. with any search domains as acquired, do not set `~.` though, -and do not disable `default-route`.) - -## The APIs - -Now we determined how we want to configure things, but how do you actually get -the configuration to `systemd-resolved.service`? There are three relevant -interfaces: - -1. Ideally, you use D-Bus and talk to [`systemd-resolved.service`'s D-Bus - API](https://www.freedesktop.org/software/systemd/man/org.freedesktop.resolve1.html) - directly. Use `SetLinkDomains()` to set the per-interface search and routing - domains on the interfaces you manage, and `SetLinkDefaultRoute()` to manage - the `default-route` boolean, all on the `org.freedesktop.resolve1.Manager` - interface of the `/org/freedesktop/resolve1` object. - -2. If that's not in the cards, you may shell out to - [`resolvectl`](https://www.freedesktop.org/software/systemd/man/resolvectl.html), - which is a thin wrapper around the D-Bus interface mentioned above. Use - `resolvectl domain …` to set the search/routing domains and - `resolvectl default-route …` to set the `default-route` boolean. - - Example use from a shell callout of your VPN software for a *corporate* VPN: - - resolvectl domain corporate0 '~corp-company.example' '~2.0.192.in-addr.arpa' - resolvectl default-route corporate0 false - resolvectl dns corporate0 192.0.2.1 - - Example use from a shell callout of your VPN software for a *privacy* VPN: - - resolvectl domain privacy0 '~.' - resolvectl default-route privacy0 true - resolvectl dns privacy0 8.8.8.8 - -3. If you don't want to use any `systemd-resolved` commands, you may use the - `resolvconf` wrapper we provide. `resolvectl` is actually a multi-call - binary and may be symlinked to `resolvconf`, and when invoked like that - behaves in a way that is largely compatible with FreeBSD's and - Ubuntu's/Debian's - [`resolvconf(8)`](https://manpages.ubuntu.com/manpages/trusty/man8/resolvconf.8.html) - tool. When the `-x` switch is specified, the `~.` routing domain is - automatically appended to the domain list configured, as appropriate for a - *privacy* VPN. Note that the `resolvconf` interface only covers *privacy* - VPNs and regular network interfaces (such as WiFi or Ethernet) well. The - *corporate* kind of VPN is not well covered, since the interface cannot - propagate the `default-route` boolean, nor can be used to configure the - `~….in-addr.arpa` or `~.ip6.arpa` routing domains. - -## Ordering - -When configuring per-interface DNS configuration settings it is wise to -configure everything *before* actually upping the interface. Once the interface -is up `systemd-resolved.service` might start using it, and hence it's important -to have everything configured properly (this is particularly relevant when -LLMNR or MulticastDNS is enabled, since that works without any explicitly -configured DNS configuration). It is also wise to configure search/routing -domains and the `default-route` boolean *before* configuring the DNS servers, -as the former without the latter has no effect, but the latter without the -former will result in DNS traffic possibly being generated, in a non-desirable -way given that the routing information is not set yet. - -## Downgrading Search Domains to Routing Domains - -Many VPN implementations provide a way how VPN servers can inform VPN clients -about search domains to use. In some cases it might make sense to install those -as routing domains instead of search domains. Unqualified domain names usually -imply a context of locality: the same unqualified name typically is expected to -resolve to one system in one local network, and to another one in a different -network. Search domains thus generally come with security implications: they -might cause that unqualified domains are resolved in a different (possibly -remote) context, contradicting user expectations. Thus it might be wise to -downgrade *search domains* provided by VPN servers to *routing domains*, so -that local unqualified name resolution remains untouched and strictly maintains -its local focus — in particular in the aforementioned less trusted *corporate* -VPN scenario. - -To illustrate this further, here's an example for an attack scenario using -search domains: a user assumes the printer system they daily contact under the -unqualified name "printer" is the network printer in their basement (with the -fully qualified domain name "printer.home"). Sometimes the user joins the -corporate VPN of their employer, which comes with a search domain -"foocorp.example", so that the user's confidential documents (maybe a job -application to a competing company) might end up being printed on -"printer.foocorp.example" instead of "printer.home". If the local VPN software -had downgraded the VPN's search domain to a routing domain "~foocorp.example", -this mismapping would not have happened. - -When connecting to untrusted WiFi networks it might be wise to go one step -further even: suppress installation of search/routing domains by the network -entirely, to ensure that the local DNS information is only used for name -resolution of qualified names and only when no better DNS configuration is -available. diff --git a/docs/ROOTFS_DISCOVERY.md b/docs/ROOTFS_DISCOVERY.md deleted file mode 100644 index 8a322a6f9e0..00000000000 --- a/docs/ROOTFS_DISCOVERY.md +++ /dev/null @@ -1,405 +0,0 @@ ---- -title: Boot Components & Root File System Discovery -category: Booting -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Boot Components & Root File System Discovery - -The recommended way to boot a [`systemd`](https://systemd.io/) based -[UEFI](https://en.wikipedia.org/wiki/UEFI) system consists primarily of three -components: - -1. A boot loader, - i.e. [`systemd-boot`](https://www.freedesktop.org/software/systemd/man/latest/systemd-boot.html) - that provides interactive and programmatic control of what precisely to - boot. It takes care of enumerating all possible boot targets (implementing - the [Boot Loader - Specification](https://uapi-group.org/specifications/specs/boot_loader_specification/)), - potentially presenting it to the user in a menu, but otherwise picking an - item automatically, implementing boot counting and automatic rollback if - desired. - -2. A [unified kernel image - ("UKI")](https://uapi-group.org/specifications/specs/unified_kernel_image/), - i.e. an UEFI PE executable that combines - [`systemd-stub`](https://www.freedesktop.org/software/systemd/man/latest/systemd-stub.html), - a Linux kernel, and an initial RAM disk ("`initrd`") into one. UKIs are - self-descriptive: the aforementioned boot loader enumerates these UKIs and - automatically extracts all information necessary to determine which menu - entries to generate for them. Within the UKI runtime (very early during - kernel initialization) the transition from the UEFI firmware code to the - Linux code takes place, i.e. it executes the fundamental - `ExitBootServices()` UEFI call that ends PC firmware control, - and lets the Linux kernel take over. - -3. A root file system ("`rootfs`"): this is where the regular OS is - located. The primary job of the early userspace that is contained in the - `initrd` that itself is part of the UKI, is to find, set up, and pivot into - the `rootfs`. - -> [!NOTE] -> The above is how `systemd` upstream recommends a system is put together. However, -> distributions differ from this, sometimes massively – in particular when -> their focus is on supporting legacy (i.e. non-UEFI) hardware platforms. We -> believe the above three components are all that's really necessary for a -> robust, simple and comprehensive system, but downstreams might see things -> differently. The above however is supposed to be a guideline for distribution -> developers. - -## Execution Environments - -Note that these three components are executed within very distinct execution -environments, with very different APIs, drivers and file system access: - -| Component | Environment | -|-----------|-------------| -| Boot Loader: `systemd-boot` | UEFI APIs, simple VFAT file system access | -| UKI initially: `systemd-stub` | UEFI APIs, simple VFAT file system access | -| UKI finally: `initrd` | Linux APIs, complex storage and file systems | -| Root File System: `rootfs` | Linux APIs, full OS functionality | - -## Structure & Auxiliary Resources - -Each of the three components is primarily encapsulated in a single object each: - -1. The boot loader is primarily a single PE UEFI binary, called either - `systemd-boot.efi` or `BOOTX64.EFI` depending on context (the latter name - contains an architecture identifier, and is different for non-x86-64 - architectures). - -2. The UKI is primarily a single PE UEFI binary (i.e. a `.efi` file). - -3. The `rootfs` is typically a Linux file system, on a GPT partition table - disk. Typically, the `rootfs` is placed within some form of container that - ensures security of the file system, i.e. authenticity, confidentiality - and integrity via `dm-verity`, `dm-crypt`, `dm-integrity` or a combination - thereof. - -While these three objects are generally enough to boot an OS successfully, in -many cases some parameterization and modularization of the boot is necessary, -hence each of these components is often combined with certain optional, -auxiliary resources: - -1. The boot loader can read a configuration file - [`loader.conf`](https://www.freedesktop.org/software/systemd/man/latest/loader.conf.html), - find additional drivers, or key material for SecureBoot enrollment in the - same file system it itself is placed in. - -2. `systemd-stub` can find additional parameters (["system - credentials"](https://systemd.io/CREDENTIALS)), configuration - (["`confext`"](https://www.freedesktop.org/software/systemd/man/latest/systemd-confext.html)), - drivers/firmware ("`sysext`") and other resources ("EFI Addons") placed next - to the location it itself is placed in. We typically call these companion - resources "sidecars". - -4. The `rootfs` often is a combination of one file system for `/usr/` - ("`usrfs`") and one for the actual root `/`, and possibly further, - auxiliary file systems, for example `/home/` or `/srv/`. - -> [!NOTE] -> Depending on the execution environment the first component (the boot loader) -> might be dispensable. Specifically, on disk images intended solely for use in -> VMs, it might be make sense to tell the firmware to directly boot a UKI, -> letting the VMM's image selection functionality play the role of the boot loader. - -> [!NOTE] -> Depending on the execution environment the last component (the `rootfs`) -> might also be dispensable. Specifically, for simpler fixed-purpose, stateless -> applications it might be sufficient to run everything needed directly from -> the `initrd` file system embedded in the UKI, and never transition out of -> this. In this case, conceptually the `initrd` is only an `initrd` from kernel -> PoV, but is already the `rootfs` from a userspace PoV. We usually call these -> types of setups Unified System Image ("USI"), as opposed to UKI. - -![Schematic Chart of Root File System Discovery](rootfs-discovery-flow.svg) - -## Automatic Discovery on Disks - -In the most common case all three components and their sidecars are placed on -the same disk. Specifically: - -1. The boot loader is placed in the "EFI System Partition" (ESP), typically at - the paths `/EFI/BOOT/BOOTX64.EFI` (this is a generic entrypoint binary that - the firmware executes when you just point it to a disk to boot without any - further details) and `/EFI/systemd/systemd-bootx64.efi` (this is a more - specific entrypoint that can be registered persistently in the firmware, to - give it an explicit starting point). The ESP is a concept defined by the - UEFI specification and is what the firmware initially looks for and - mounts. Since VFAT is the only relevant file system type UEFI firmwares have - to support the ESP is generally a VFAT file system. The aforementioned - auxiliary, optional resources the boot loader may consume are placed in the - ESP as well, in particular below the `/loader/` subdirectory. - -2. The UKIs may either be placed in the ESP (below the `/EFI/Linux/` - subdirectory), or in the [Extended Boot Loader - Partition](https://uapi-group.org/specifications/specs/boot_loader_specification/#the-partitions) - ("XBOOTLDR"), which can be placed on the same disk as the ESP and is also - VFAT. XBOOTLDR is an optional concept and it's only *raison d'être* is that - ESPs sometimes are sized too small by vendors, and do not have enough space - for multiple UKIs. XBOOTLDR hence serves as a conceptual extension of the - size-constrained ESP. Sidecars for the UKIs are typically placed in a - directory next to the UKI they are for, whose name however is suffixed by - `.d/`, i.e. a UKI `foo.efi` has its sidecars in `foo.efi.d/`. - -3. The `rootfs` is placed on the same disk as the ESP/XBOOTLDR, in a partition - marked with a special GPT partition type. Various other well-known types of - partitions can be placed next to the `rootfs` and are automatically - discovered and mounted, see the [Discoverable Partitions - Specification](https://uapi-group.org/specifications/specs/discoverable_partitions_specification/) - for details. - -In this common case, discovery of all three components and their sidecars is -fully automatic. Each component derives automatically where to find its -auxiliary resources as well as the next step to transition to, entirely based -on the place itself is running from. There's a full chain of automatic -discovery in place: - -1. The firmware picks the disk to boot from (possibly by interactive choice of - the user), accesses the ESP on it, and invokes the boot loader from it. - -2. The boot loader then looks for UKIs, both on the ESP it was invoked from, - and in the XBOOTLDR partition next to it on the same disk. - -3. The UKI's initrd then looks for the `rootfs`, on the same disk the - UKI was invoked from, i.e. it looks for a partition marked as root next to - the ESP/XBOOTLDR partition. (This information is passed from UKI to - userspace via the - [`LoaderDevicePartUUID`](https://systemd.io/BOOT_LOADER_INTERFACE) EFI - variable.) - -In more complex setups it is possible to specify in more detail where to find -each of these resources: - -1. Firmware typically provides a basic boot menu which may be used to choose - between various relevant boot loaders/entrypoints on multiple disks. This - is sometimes configurable from the firmware setup tool, as well as from - userspace via tools such as - [`bootctl`](https://www.freedesktop.org/software/systemd/man/latest/bootctl.html), - `efibootmgr` or `kernel-bootcfg`. - -2. The `systemd-boot` boot loader may be configured via [`Boot Loader - Specification Type #1`](https://uapi-group.org/specifications/specs/boot_loader_specification/) - entries to acquire UKIs or similar from other locations. - -3. The `initrd` part of the UKI understands the `root=` (and `mount.usr=`) - kernel command line switches to look for the `rootfs`/`usrfs` at a - particular place. - -While it is recommended to keep all three components closely together it is -possible via these mechanisms to place all three at completely disparate -locations, too. - -## Network Boot - -In many cases it is essential to boot an OS from the network instead of a -local disk. This can happen at each of these three components: - -1. Many UEFI firmwares support HTTP(S) network boot (usually requires enabling - in firmware setup). If this is available, it permits downloading a disk image - from an HTTP server (the URL can either be configured in the firmware setup, - or be acquired in a DHCP lease). The disk image is then set up as a RAM - disk, and then processed much like a regular disk: an ESP is searched for - and the `/EFI/BOOT/BOOTX64.EFI` entrypoint binary is invoked. - -2. UKIs can be placed on the same downloaded disk image, within the ESP. If - multiple different UKIs shall be made accessible from the same boot menu - this would potentially increase the size of the disk image to prohibitive sizes. - In order to address this, it is possible to embed Boot - Loader Specification Type #1 entry files in the ESP instead, which may carry - references to the UKIs to download and invoke once a choice is made. These - references can either be full URLs or alternatively simple filenames which - are then automatically appended to the URL that was used by the firmware - to acquire the initial boot disk. - -3. The `rootfs` can be acquired automatically from a networked source too in a - flexible fashion. For example, the `initrd` contained in the UKI might - support NVMe-over-TCP or iSCSI block devices to boot from, supporting the - whole Linux storage stack. `systemd` also natively [supports downloading the - `rootfs` from HTTP - sources](https://www.freedesktop.org/software/systemd/man/latest/systemd-import-generator.html), - either in a GPT disk image (specifically: - [DDIs](https://uapi-group.org/specifications/specs/discoverable_disk_image/), - with `.raw` suffix) or in a `.tar` file, which are placed in system RAM and - then booted into (these downloads can be downloaded in compressed form and - are automatically decompressed on-the-fly). This of course requires - sufficient RAM to be available on the target system, and also means that - persistency of modifications of the file system is not possible. If this - mode is used, the URL to acquire the `rootfs` disk image from can be derived - automatically from the URL that was used to acquire the UKI itself. (This - information is passed from UKI to userspace via the `LoaderDevicePartURL` - EFI variable.) - -Similarly to the disk-based boot scheme described in the previous section, -discovery of the boot source can be fully automatic, with each -component taking the source of the preceding component into account: -the boot loader can automatically download UKIs from the same source -it itself was downloaded from. Moreover the `initrd` of the UKI can -automatically downloads the `rootfs` from the same source it itself was -downloaded from. - -Also, much like in the disk-based boot scheme, it is possible to -specify a different source for a component to replace the automatically-derived URL. -On top of that it is of course possible to mix disk-based and network-based boot: -for example place the boot loader on the local disk, -but use UKIs and `rootfs` from networked sources; -or alternatively place both boot loader and UKIs on the local disk, -and only the `rootfs` on the network. - -## Trust & Security - -In a modern world of boot integrity, all three of the relevant components as well -as (most of) their sidecars require cryptographic protection. Specifically: - -1. The boot loader is typically authenticated by the firmware before invocation - via UEFI Secure Boot, i.e. checked against a cryptographic certificate list - persistently stored in the firmware. Note that the various auxiliary - resources the `systemd-boot` boot loader reads are not individually - authenticated (i.e. `loader.conf` as well as Type #1 Boot Loader - Specification entries). Because of this they can typically only be used in a - very restricted fashion, i.e. configure some UI details as well as menu - entries. Some options available are ignored if Secure Boot mode is - enabled. Moreover, even if the text strings shown in the menu entries might - not be authenticated, the binaries that are invoked once they are selected - are, as are all their parameters. - -2. The UKIs are also authenticated by the firmware via UEFI Secure Boot, and so - are EFI Addons. `confext` and `sysext` sidecars are protected via - `dm-verity` and a signature of the root hash is validated against keys in - the kernel's keyrings. System credentials are authenticated via secrets - stored in the TPM. - -3. Authentication of the `rootfs` and `usrfs` is more variable: depending on - setup this is either done via `dm-verity` (either pinned by root hash from - the UKI, or authenticated by signature provided to the kernel, checked - against the kernel's built-in keyring) or `dm-crypt`+`dm-integrity` - (protected by TPM or user provided password/FIDO/PKCS#11), or in the network - case at download time via detached signatures (currently only GPG) or via - HTTPS certificate validation. Note that by default the automatic discovery - mechanism of the `rootfs` and its auxiliary file systems does not insist on - cryptographic protection and authentication before use. However, the - [`systemd.image_policy=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.image-policy.html) - kernel command line switch may be used to control precisely what kind of - protection to require for each such partition. - -Note that UEFI Secure Boot is problematic in various ways: it is generally -bound to a certificate list maintained centrally by Microsoft, and thus implies -a complex (and expensive) code signing bureaucracy, that in many cases is -undesirable, particularly in a community Linux world. Moreover, because the -certificate list managed by Microsoft is very large, its security value is -limited: it mostly acts more as denylist of known-bad software rather than as -allowlist of known-good. (If you enroll your own list, things are much better, -but see below.) - -### Shim - -To make the code signing more palatable to the Linux world the `shim` project -has been developed, which is often used as initial component of the OS boot -(i.e. the firmware would invoke `shim` as component 0, before the components 1, -2, 3 described above). `shim` is primarily relevant for two reasons: it adds -a second set of certificates on top of the UEFI Secure Boot list, maintained by -the OS vendor, and it optionally provides functionality to maintain a local set -of keys ("MOK") in addition to the Microsoft and OS vendor keys. - -### Automatic Enrollment of Secure Boot Certificates - -The `systemd-boot` boot loader also supports automatic enrollment of -alternative SecureBoot certificates: if the system is booted in the -firmware-provided Secure Boot "Setup Mode", it can automatically enroll -certificates placed inside the ESP into the firmware, replacing any existing -ones if there are any. This mechanism massively enhances the security value of -Secure Boot: you can enroll your own certificates, ensuring that only the -software you want shall be allowed to be run on the system, in a very focused -way. However, do note that this mechanism is only suitable if the hardware -supports it properly, because the Secure Boot certificate -list is also used to authenticate firmware extensions provided by certain -extension boards of PCs (for example graphics cards). Or in other words: -replacing the certificate list with your own might result in unbootable and even -bricked systems. Automatic enrolling of Secure Boot certificates is however a -really good option if the targeted hardware is known to be compatible, -which is in particular the case in VMs. - -### Measured Boot and TPMs - -Secure Boot is not the only mechanism that can provide boot time integrity -guarantees of the OS. Most modern systems are equipped with a TPM security -chip. It allows components of the boot to issue "measurements" (i.e. submit -a cryptographic hash) of the next step of the boot process as well as of all -inputs they consume to the device, in a fashion that cannot be undone (except -if the system is rebooted). The combination of measurements of all such boot -components can then later be used to protect secrets the TPM can manage: only -if the system is booted in a very specific way such secrets (such as a disk -encryption key) can be revealed to the OS. This hence provides a different form -of protection: instead of making it *a-priori* impossible to boot or consume -untrusted components (as Secure Boot would do it), anything is permitted, -however the TPM would never reveal protected secrets to the OS unless the -components are trusted, in a *a-posteriori* fashion. This generally provides a -more focused security model (as the list of allowed components and the -policies derived thereof are locally maintained instead of world-wide by -Microsoft), however, requires more careful management of OS and firmware -updates. Moreover, it's more compatible with a TOFU security model ("Trust on -first use") rather than a universal trust model. - -`systemd-stub` will measure the sidecars it picks up as well as the individual -parts that make up a UKI that are used for boot. `systemd` userspace will also -measure various parts of its resources as does the kernel. - -[`systemd-pcrlock`](https://www.freedesktop.org/software/systemd/man/latest/systemd-pcrlock.html), -[`systemd-measure`](https://www.freedesktop.org/software/systemd/man/latest/systemd-measure.html), -[`systemd-cryptenroll`](https://www.freedesktop.org/software/systemd/man/latest/systemd-cryptenroll.html), -[`systemd-cryptsetup`](https://www.freedesktop.org/software/systemd/man/latest/systemd-cryptsetup.html) -can be used to manage Measured Boot policies for disk encryption. - -Note that Secure Boot and Measured Boot are not exclusive to each other, they -are often used in combination, and can interact (e.g. the Secure Boot -certificate lists are measured as part of the boot process). - -### Security of Network Boot - -UEFI HTTP boot comes in two flavours: plain HTTP and HTTPS. The latter -typically requires enrolment of TLS server certificates in the system -firmware, but provides transport integrity, authenticity and -confidentiality. Acquiring the various resources via plain HTTP should -generally be sufficient too, as the key resources acquired this way -area generally authenticated before use via other mechanisms, see above. - -## Building - -The `systemd` project provides various tools to build the various components -necessary to implement the aforementioned boot process: - -* The `systemd-boot`, `systemd-stub` components are part of the `systemd` - source tree. - -* The - [`ukify`](https://www.freedesktop.org/software/systemd/man/latest/ukify.html) - tool provided by `systemd` can be used to build UKIs, USIs and EFI Addons. - -* The `bootctl` tool provided by `systemd` can be used to install the - `systemd-boot` boot loader to the ESP. - -* The - [`kernel-install`](https://www.freedesktop.org/software/systemd/man/latest/kernel-install.html) - tool provided by `systemd` can be used to install UKIs to the ESP or - XBOOTLDR. - -* The - [`systemd-repart`](https://www.freedesktop.org/software/systemd/man/latest/systemd-repart.html) - tool can be used to generate `confext` and `sysext` images, as well as - `rootfs` and `usrfs` images. It can also be used install such images on other - disks, or to augment minimal disk images on boot with additional partitions, - or grow them. - -* The - [`systemd-creds`](https://www.freedesktop.org/software/systemd/man/latest/systemd-creds.html) - tool can be used to generate system credential files. - -* The `systemd-cryptsetup`, `systemd-veritysetup` and `systemd-integritysetup` - tools can be used to set up cryptographically protected disks at boot. - -* [`mkosi`](https://github.com/systemd/mkosi) is a higher level tool that - combines all of the above to build and sign complete OS images from - distribution packages, as needed: `initrd` trees, UKIs, USIs, - `rootfs`/`usrfs`, whole DDIs, `sysext` images, `.tar` files to boot into and - more. diff --git a/docs/SEPARATE_USR_IS_BROKEN.md b/docs/SEPARATE_USR_IS_BROKEN.md deleted file mode 100644 index 56129450d17..00000000000 --- a/docs/SEPARATE_USR_IS_BROKEN.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: Booting Without /usr is Broken -category: Manuals and Documentation for Users and Administrators -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Booting Without /usr is Broken - -You probably discovered this page because your shiny new systemd system referred you here during boot time, -when it warned you that booting without `/usr` pre-mounted wasn't supported anymore. -And now you wonder what this all is about. -Here's an attempt of an explanation: - -One thing in advance: -systemd itself is actually mostly fine with `/usr` on a separate file system that is not pre-mounted at boot time. -However, the common basic set of OS components of modern Linux machines is not, and has not been in quite some time. -And it is unlikely that this is going to be fixed any time soon, or even ever. - -Most of the failures you will experience with `/usr` split off and not pre-mounted in the initramfs are graceful failures: -they won't become directly visible, however certain features become unavailable due to these failures. -Quite a number of programs these days hook themselves into the early boot process at various stages. -A popular way to do this is for example via udev rules. -The binaries called from these rules are sometimes located on `/usr/bin`, or link against libraries in `/usr/lib`, -or use data files from `/usr/share`. -If these rules fail udev will proceed with the next one, -however later on applications will then not properly detect these udev devices or features of these devices. -Here's a short, very in-comprehensive list of software we are aware of that currently are not able to provide the full set of functionality when `/usr` is split off and not pre-mounted at boot: -udev-pci-db/udev-usb-db and all rules depending on this -(using the PCI/USB database in `/usr/share`), -PulseAudio, NetworkManager, ModemManager, udisks, libatasmart, usb\_modeswitch, -gnome-color-manager, usbmuxd, ALSA, D-Bus, CUPS, Plymouth, LVM, hplip, multipath, Argyll, VMWare, -the locale logic of most programs and a lot of other stuff. - -You don't believe us? -Well, here's a command line that reveals a few obvious cases of udev rules that will silently fail to work if `/usr` is split off and not pre-mounted: -`egrep 'usb-db|pci-db|FROM_DATABASE|/usr' /*/udev/rules.d/*` --- and you find a lot more if you actually look for it. -On my fresh Fedora 15 install that's 23 obvious cases. - -## The Status Quo - -Due to this, many upstream developers have decided to consider the problem of a separate -`/usr` that is not mounted during early boot an outdated question, -and started to close bugs regarding these issues as WONTFIX. -We certainly cannot blame them, as the benefit of supporting this is questionable and brings a lot of additional work with it. - -And let's clarify a few things: - -1. **It isn't systemd's fault.** systemd mostly works fine with `/usr` on a separate file system that is not pre-mounted at boot. -2. **systemd is merely the messenger.** Don't shoot the messenger. -3. **There's no news in all of this.** The message you saw is just a statement of fact, describing the status quo. - Things have been this way since a while. -4. **The message is merely a warning.** You can choose to ignore it. -5. **Don't blame us**, don't abuse us, it's not our fault. -We have been working on the Linux userspace since quite some time, -and simply have enough of the constant bug reports regarding these issues, -since they are actually very hard to track down because the failures are mostly graceful. -Hence we placed this warning into the early boot process of every systemd Linux system with a split off and not pre-mounted -`/usr`, so that people understand what is going on. - -## Going Forward - -`/usr` on its own filesystem is useful in some custom setups. -But instead of expecting the traditional Unix way to (sometimes mindlessly) distributing tools between `/usr` and `/`, -and require more and more tools to move to `/`, -we now just expect `/usr` to be pre-mounted from inside the initramfs, to be available before 'init' starts. -The duty of the minimal boot system that consisted of `/bin`, `/sbin` and `/lib` on traditional Unix, -has been taken over by the initramfs of modern Linux. -An initramfs that supports mounting `/usr` on top of `/` before it starts 'init', makes all existing setups work properly. - -There is no way to reliably bring up a modern system with an empty `/usr`. -There are two alternatives to fix it: move `/usr` back to the rootfs or use an initramfs which can hide the split-off from the system. - -On the Fedora distribution we have succeeded to clean up the situation and the confusion the current split between `/` and `/usr` has created. -We have moved all tools that over time have been moved to `/` back to `/usr` (where they belong), -and the root file system only contains compatibility symlinks for `/bin` and `/sbin` into `/usr`. -All binaries of the system are exclusively located within the `/usr` hierarchy. - -In this new definition of `/usr`, the directory can be mounted read-only by default, -while the rootfs may be either read-write or read-only (for stateless systems) and contains only the empty mount point directories, -compat-symlinks to `/usr` and the host-specific data like `/etc`, `/root`, `/srv`. -In comparison to today's setups, the rootfs will be very small. -The host-specific data will be properly separated from the installed operating system. -The new `/usr` could also easily be shared read-only across several systems. -Such a setup would be more efficient, can provide additional security, is more flexible to use, -provides saner options for custom setups, and is much simpler to setup and maintain. - -For more information on this please continue to [The Case for the /usr Merge](/THE_CASE_FOR_THE_USR_MERGE). diff --git a/docs/SYSLOG.md b/docs/SYSLOG.md deleted file mode 100644 index 9999a8dd8d7..00000000000 --- a/docs/SYSLOG.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -title: Writing syslog Daemons Which Cooperate Nicely With systemd -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Writing syslog Daemons Which Cooperate Nicely With systemd - -Here are a few notes on things to keep in mind when you work on a classic BSD syslog daemon for Linux, to ensure that your syslog daemon works nicely together with systemd. -If your syslog implementation does not follow these rules, then it will not be compatible with systemd v38 and newer. - -A few notes in advance: systemd centralizes all log streams in the Journal daemon. -Messages coming in via /dev/log, via the native protocol, via STDOUT/STDERR of all services and via the kernel are received in the journal daemon. - -The journal daemon then stores them to disk or in RAM (depending on the configuration of the Storage= option in journald.conf), and optionally forwards them to the console, the kernel log buffer, or to a classic BSD syslog daemon -- and that's where you come in. - -Note that it is now the journal that listens on /dev/log, no longer the BSD syslog daemon directly. -If your logging daemon wants to get access to all logging data then it should listen on /run/systemd/journal/syslog instead via the syslog.socket unit file that is shipped along with systemd. -On a systemd system it is no longer OK to listen on /dev/log directly, and your daemon may not bind to the /run/systemd/journal/syslog socket on its own. -If you do that then you will lose logging from STDOUT/STDERR of services (as well as other stuff). - -Your BSD compatible logging service should alias `syslog.service` to itself (i.e. symlink) when it is _enabled_. -That way [syslog.socket](http://cgit.freedesktop.org/systemd/systemd/plain/units/syslog.socket) will activate your service when things are logged. -Of course, only one implementation of BSD syslog can own that symlink, and hence only one implementation can be enabled at a time, but that's intended as there can only be one process listening on that socket. -(see below for details how to manage this symlink.) - -Note that this means that syslog.socket as shipped with systemd is _shared_ among all implementations, and the implementation that is in control is configured with where syslog.service points to. - -Note that journald tries hard to forward to your BSD syslog daemon as much as it can. -That means you will get more than you traditionally got on /dev/log, such as stuff all daemons log on STDOUT/STDERR and the messages that are logged natively to systemd. Also, we will send stuff like the original SCM_CREDENTIALS along if possible. - -(BTW, journald is smart enough not to forward the kernel messages it gets to you, you should read that on your own, directly from /proc/kmsg, as you always did. -It's also smart enough never to forward kernel messages back to the kernel, but that probably shouldn't concern you too much...) - -And here are the recommendations: - -- First of all, make sure your syslog daemon installs a native service unit file (SysV scripts are not sufficient!) and is socket activatable. Newer systemd versions (v35+) do not support non-socket-activated syslog daemons anymore and we do no longer recommend people to order their units after syslog.target. -That means that unless your syslog implementation is socket activatable many services will not be able to log to your syslog implementation and early boot messages are lost entirely to your implementation. -Note that your service should install only one unit file, and nothing else. Do not install socket unit files. - -- Make sure that in your unit file you set StandardOutput=null in the [Service] block. -This makes sure that regardless what the global default for StandardOutput= is the output of your syslog implementation goes to /dev/null. -This matters since the default StandardOutput= value for all units can be set to syslog and this should not create a feedback loop with your implementation where the messages your syslog implementation writes out are fed back to it. -In other words: you need to explicitly opt out of the default standard output redirection we do for other services. -(Also note that you do not need to set StandardError= explicitly, since that inherits the setting of StandardOutput= by default) - -- /proc/kmsg is your property, flush it to disk as soon as you start up. - -- Name your service unit after your daemon (e.g. rsyslog.service or syslog-ng.service) and make sure to include Alias=syslog.service in your [Install] section in the unit file. -This is ensures that the symlink syslog.service is created if your service is enabled and that it points to your service. -Also add WantedBy=multi-user.target so that your service gets started at boot, and add Requires=syslog.socket in [Unit] so that you pull in the socket unit. - -Here are a few other recommendations, that are not directly related to systemd: - -- Make sure to read the priority prefixes of the kmsg log messages the same way like from normal userspace syslog messages. -When systemd writes to kmsg it will prefix all messages with valid priorities which include standard syslog facility values. OTOH for kernel messages the facility is always 0. -If you need to know whether a message originated in the kernel rely on the facility value, not just on the fact that you read the message from /proc/kmsg! A number of userspace applications write messages to kmsg (systemd, udev, dracut, others), and they'll nowadays all set correct facility values. - -- When you read a message from the socket use SCM_CREDENTIALS to get information about the client generating it, and possibly patch the message with this data in order to make it impossible for clients to fake identities. - -The unit file you install for your service should look something like this: - -``` -[Unit] -Description=System Logging Service -Requires=syslog.socket - -[Service] -ExecStart=/usr/sbin/syslog-ng -n -StandardOutput=null - -[Install] -Alias=syslog.service -WantedBy=multi-user.target -``` - -And remember: don't ship any socket unit for /dev/log or /run/systemd/journal/syslog (or even make your daemon bind directly to these sockets)! That's already shipped along with systemd for you. diff --git a/docs/SYSTEMD_FILE_HIERARCHY_REQUIREMENTS.md b/docs/SYSTEMD_FILE_HIERARCHY_REQUIREMENTS.md deleted file mode 100644 index 81d6f437d74..00000000000 --- a/docs/SYSTEMD_FILE_HIERARCHY_REQUIREMENTS.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: systemd File Hierarchy Requirements -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# systemd File Hierarchy Requirements - -There are various attempts to standardize the file system hierarchy of Linux systems. -In systemd we leave much of the file system layout open to the operating system, but here's what systemd strictly requires: - -- `/`, `/usr`, `/etc` must be mounted when the host systemd is first invoked. - This may be achieved either by using the kernel's built-in root disk mounting (in which case `/`, `/usr` and `/etc` need to be on the same file system), or via an initrd, which could mount the three directories from different sources. - -- `/bin`, `/sbin`, `/lib` (and `/lib64` if applicable) should reside on `/`, or be symlinks to the `/usr` file system (recommended). - All of them must be available before the host systemd is first executed. - -- `/var` does not have to be mounted when the host systemd is first invoked, however, - it must be configured so that it is mounted writable before local-fs.target is reached (for example, by simply listing it in` /etc/fstab`). - -- `/tmp` is recommended to be a tmpfs (default), but doesn't have to. - If configured, it must be mounted before local-fs.target is reached (for example, by listing it in `/etc/fstab`). - -- `/dev` must exist as an empty mount point and will automatically be mounted by systemd with a devtmpfs. Non-devtmpfs boots are not supported. - -- `/proc` and `/sys` must exist as empty mount points and will automatically be mounted by systemd with procfs and sysfs. - -- `/run` must exist as an empty mount point and will automatically be mounted by systemd with a tmpfs. - -The other directories usually found in the root directory (such as `/home`, `/boot`, `/opt`) are irrelevant to systemd. -If they are defined they may be mounted from any source and at any time, though it is a good idea to mount them also before local-fs.target is reached. diff --git a/docs/THE_CASE_FOR_THE_USR_MERGE.md b/docs/THE_CASE_FOR_THE_USR_MERGE.md deleted file mode 100644 index 30901eb7acd..00000000000 --- a/docs/THE_CASE_FOR_THE_USR_MERGE.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -title: The Case for the /usr Merge -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# The Case for the /usr Merge - -**Why the /usr Merge Makes Sense for Compatibility Reasons** - -_This is based on the [Fedora feature](https://fedoraproject.org/wiki/Features/UsrMove) for the same topic, put together by Harald Hoyer and Kay Sievers. This feature has been implemented successfully in Fedora 17._ - -Note that this page discusses a topic that is actually independent of systemd. systemd supports both systems with split and with merged /usr, and the /usr merge also makes sense for systemd-less systems. That said we want to encourage distributions adopting systemd to also adopt the /usr merge. - -### What's Being Discussed Here? - -Fedora (and other distributions) have finished work on getting rid of the separation of /bin and /usr/bin, as well as /sbin and /usr/sbin, /lib and /usr/lib, and /lib64 and /usr/lib64. All files from the directories in / will be merged into their respective counterparts in /usr, and symlinks for the old directories will be created instead: - -``` -/bin → /usr/bin -/sbin → /usr/sbin -/lib → /usr/lib -/lib64 → /usr/lib64 -``` - -You are wondering why merging /bin, /sbin, /lib, /lib64 into their respective counterparts in /usr makes sense, and why distributions are pushing for it? You are wondering whether your own distribution should adopt the same change? Here are a few answers to these questions, with an emphasis on a compatibility point of view: - -### Compatibility: The Gist of It - -- Improved compatibility with other Unixes/Linuxes in _behavior_: After the /usr merge all binaries become available in both /bin and /usr/bin, resp. both /sbin and /usr/sbin (simply because /bin becomes a symlink to /usr/bin, resp. /sbin to /usr/sbin). That means scripts/programs written for other Unixes or other Linuxes and ported to your distribution will no longer need fixing for the file system paths of the binaries called, which is otherwise a major source of frustration. /usr/bin and /bin (resp. /usr/sbin and /sbin) become entirely equivalent. -- Improved compatibility with other Unixes (in particular Solaris) in _appearance_: The primary commercial Unix implementation is nowadays Oracle Solaris. Solaris has already completed the same /usr merge in Solaris 11. By making the same change in Linux we minimize the difference towards the primary Unix implementation, thus easing portability from Solaris. -- Improved compatibility with GNU build systems: The biggest part of Linux software is built with GNU autoconf/automake (i.e. GNU autotools), which are unaware of the Linux-specific /usr split. Maintaining the /usr split requires non-trivial project-specific handling in the upstream build system, and in your distribution's packages. With the /usr merge, this work becomes unnecessary and porting packages to Linux becomes simpler. -- Improved compatibility with current upstream development: In order to minimize the delta from your Linux distribution to upstream development the /usr merge is key. - -### Compatibility: The Longer Version - -A unified filesystem layout (as it results from the /usr merge) is more compatible with UNIX than Linux’ traditional split of /bin vs. /usr/bin. Unixes differ in where individual tools are installed, their locations in many cases are not defined at all and differ in the various Linux distributions. The /usr merge removes this difference in its entirety, and provides full compatibility with the locations of tools of any Unix via the symlink from /bin to /usr/bin. - -#### Example - -- /usr/bin/foo may be called by other tools either via /usr/bin/foo or /bin/foo, both paths become fully equivalent through the /usr merge. The operating system ends up executing exactly the same file, simply because the symlink /bin just redirects the invocation to /usr/bin. - -The historical justification for a /bin, /sbin and /lib separate from /usr no longer applies today. ([More on the historical justification for the split](http://lists.busybox.net/pipermail/busybox/2010-December/074114.html), by Rob Landley) They were split off to have selected tools on a faster hard disk (which was small, because it was more expensive) and to contain all the tools necessary to mount the slower /usr partition. Today, a separate /usr partition already must be mounted by the initramfs during early boot, thus making the justification for a split-off moot. In addition a lot of tools in /bin and /sbin in the status quo already lost the ability to run without a pre-mounted /usr. There is no valid reason anymore to have the operating system spread over multiple hierarchies, it lost its purpose. - -Solaris implemented the core part of the /usr merge 15 years ago already, and completed it with the introduction of Solaris 11. Solaris has /bin and /sbin only as symlinks in the root file system, the same way as you will have after the /usr merge: [Transitioning From Oracle Solaris 10 to Oracle Solaris 11 - User Environment Feature Changes](http://docs.oracle.com/cd/E23824_01/html/E24456/userenv-1.html). - -Not implementing the /usr merge in your distribution will isolate it from upstream development. It will make porting of packages needlessly difficult, because packagers need to split up installed files into multiple directories and hard code different locations for tools; both will cause unnecessary incompatibilities. Several Linux distributions are agreeing with the benefits of the /usr merge and are already in the process to implement the /usr merge. This means that upstream projects will adapt quickly to the change, those making portability to your distribution harder. - -### Beyond Compatibility - -One major benefit of the /usr merge is the reduction of complexity of our system: the new file system hierarchy becomes much simpler, and the separation between (read-only, potentially even immutable) vendor-supplied OS resources and users resources becomes much cleaner. As a result of the reduced complexity of the hierarchy, packaging becomes much simpler too, since the problems of handling the split in the .spec files go away. - -The merged directory /usr, containing almost the entire vendor-supplied operating system resources, offers us a number of new features regarding OS snapshotting and options for enterprise environments for network sharing or running multiple guests on one host. Static vendor-supplied OS resources are monopolized at a single location, that can be made read-only easily, either for the whole system or individually for each service. Most of this is much harder to accomplish, or even impossible, with the current arbitrary split of tools across multiple directories. - -_With all vendor-supplied OS resources in a single directory /usr they may be shared atomically, snapshots of them become atomic, and the file system may be made read-only as a single unit._ - -#### Example: /usr Network Share - -- With the merged /usr directory we can offer a read-only export of the vendor supplied OS to the network, which will contain almost the entire operating system resources. The client hosts will then only need a minimal host-specific root filesystem with symlinks pointing into the shared /usr filesystem. From a maintenance perspective this is the first time where sharing the operating system over the network starts to make sense. Without the merged /usr directory (like in traditional Linux) we can only share parts of the OS at a time, but not the core components of it that are located in the root file system. The host-specific root filesystem hence traditionally needs to be much larger, cannot be shared among client hosts and needs to be updated individually and often. Vendor-supplied OS resources traditionally ended up "leaking" into the host-specific root file system. - -#### Example: Multiple Guest Operating Systems on the Same Host - -- With the merged /usr directory, we can offer to share /usr read-only with multiple guest operating systems, which will shrink down the guest file system to a couple of MB. The ratio of the per-guest host-only part vs. the shared operating system becomes nearly optimal. - In the long run the maintenance burden resulting of the split-up tools in your distribution, and hard-coded deviating installation locations to distribute binaries and other packaged files into multiple hierarchies will very likely cause more trouble than the /usr merge itself will cause. - -## Myths and Facts - -**Myth #1**: Fedora is the first OS to implement the /usr merge - -**Fact**: Oracle Solaris has implemented the /usr merge in parts 15 years ago, and completed it in Solaris 11. Fedora is following suit here, it is not the pioneer. - -**Myth #2**: Fedora is the only Linux distribution to implement the /usr merge - -**Fact**: Multiple other Linux distributions have been working in a similar direction. - -**Myth #3**: The /usr merge decreases compatibility with other Unixes/Linuxes - -**Fact**: By providing all binary tools in /usr/bin as well as in /bin (resp. /usr/sbin + /sbin) compatibility with hard coded binary paths in scripts is increased. When a distro A installs a tool “foo” in /usr/bin, and distro B installs it in /bin, then we’ll provide it in both, thus creating compatibility with both A and B. - -**Myth #4**: The /usr merge’s only purpose is to look pretty, and has no other benefits - -**Fact**: The /usr merge makes sharing the vendor-supplied OS resources between a host and networked clients as well as a host and local lightweight containers easier and atomic. Snapshotting the OS becomes a viable option. The /usr merge also allows making the entire vendor-supplied OS resources read-only for increased security and robustness. - -**Myth #5**: Adopting the /usr merge in your distribution means additional work for your distribution's package maintainers - -**Fact**: When the merge is implemented in other distributions and upstream, not adopting the /usr merge in your distribution means more work, adopting it is cheap. - -**Myth #6**: A split /usr is Unix “standard”, and a merged /usr would be Linux-specific - -**Fact**: On SysV Unix /bin traditionally has been a symlink to /usr/bin. A non-symlinked version of that directory is specific to non-SysV Unix and Linux. - -**Myth #7**: After the /usr merge one can no longer mount /usr read-only, as it is common usage in many areas. - -**Fact**: Au contraire! One of the reasons we are actually doing this is to make a read-only /usr more thorough: the entire vendor-supplied OS resources can be made read-only, i.e. all of what traditionally was stored in /bin, /sbin, /lib on top of what is already in /usr. - -**Myth #8**: The /usr merge will break my old installation which has /usr on a separate partition. - -**Fact**: This is perfectly well supported, and one of the reasons we are actually doing this is to make placing /usr of a separate partition more thorough. What changes is simply that you need to boot with an initrd that mounts /usr before jumping into the root file system. Most distributions rely on initrds anyway, so effectively little changes. - -**Myth #9**: The /usr split is useful to have a minimal rescue system on the root file system, and the rest of the OS on /usr. - -**Fact**: On Fedora the root directory contains ~450MB already. This hasn't been minimal since a long time, and due to today's complex storage and networking technologies it's unrealistic to ever reduce this again. In fact, since the introduction of initrds to Linux the initrd took over the role as minimal rescue system that requires only a working boot loader to be started, but not a full file system. - -**Myth #10**: The status quo of a split /usr with mounting it without initrd is perfectly well supported right now and works. - -**Fact**: A split /usr without involvement of an initrd mounting it before jumping into the root file system [hasn't worked correctly since a long time](/SEPARATE_USR_IS_BROKEN). - -**Myth #11**: Instead of merging / into /usr it would make a lot more sense to merge /usr into /. - -**Fact**: This would make the separation between vendor-supplied OS resources and machine-specific even worse, thus making OS snapshots and network/container sharing of it much harder and non-atomic, and clutter the root file system with a multitude of new directories. - ---- - -If this page didn't answer your questions you may continue reading [on the Fedora feature page](https://fedoraproject.org/wiki/Features/UsrMove) and this [mail from Lennart](http://thread.gmane.org/gmane.linux.redhat.fedora.devel/155511/focus=155792). diff --git a/docs/TRANSLATORS.md b/docs/TRANSLATORS.md deleted file mode 100644 index 2f578cc6f58..00000000000 --- a/docs/TRANSLATORS.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -title: Notes for Translators -category: Contributing -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Notes for Translators - -systemd depends on the `gettext` package for multilingual support. - -You'll find the i18n files in the `po/` directory. - -The build system (meson/ninja) can be used to generate a template (`*.pot`), -which can be used to create new translations. - -It can also merge the template into the existing translations (`*.po`), to pick -up new strings in need of translation. - -Finally, it is able to compile the translations (to `*.gmo` files), so that -they can be used by systemd software. (This step is also useful to confirm the -syntax of the `*.po` files is correct.) - -## Creating a New Translation - -To create a translation to a language not yet available, start by creating the -initial template: - -``` -$ ninja -C build/ systemd-pot -``` - -This will generate file `po/systemd.pot` in the source tree. - -Then simply copy it to a new `${lang_code}.po` file, where -`${lang_code}` is the two-letter code for a language -(possibly followed by a two-letter uppercase country code), according to the -ISO 639 standard. - -In short: - -``` -$ cp po/systemd.pot po/${lang_code}.po -``` - -Then edit the new `po/${lang_code}.po` file (for example, -using the `poedit` GUI editor.) - -## Updating an Existing Translation - -Start by updating the `*.po` files from the latest template: - -``` -$ ninja -C build/ systemd-update-po -``` - -This will touch all the `*.po` files, so you'll want to pay attention when -creating a git commit from this change, to only include the one translation -you're actually updating. - -Edit the `*.po` file, looking for empty translations and translations marked as -"fuzzy" (which means the merger found a similar message that needs to be -reviewed as it's expected not to match exactly.) - -You can use any text editor to update the `*.po` files, but a good choice is -the `poedit` editor, a graphical application specifically designed for this -purpose. - -Once you're done, create a git commit for the update of the `po/*.po` file you -touched. Remember to undo the changes to the other `*.po` files (for instance, -using `git checkout -- po/` after you commit the changes you do want to keep.) - -## Recompiling Translations - -You can recompile the `*.po` files using the following command: - -``` -$ ninja -C build/ systemd-gmo -``` - -The resulting files will be saved in the `build/po/` directory. diff --git a/docs/USERDB_AND_DESKTOPS.md b/docs/USERDB_AND_DESKTOPS.md deleted file mode 100644 index b9a9eace590..00000000000 --- a/docs/USERDB_AND_DESKTOPS.md +++ /dev/null @@ -1,161 +0,0 @@ ---- -title: systemd-homed and JSON User/Group Record Support in Desktop Environments -category: Users, Groups and Home Directories -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# `systemd-homed` and JSON User/Group Record Support in Desktop Environments - -Starting with version 245, systemd supports a new subsystem -[`systemd-homed.service`](https://www.freedesktop.org/software/systemd/man/systemd-homed.service.html) -for managing regular ("human") users and their home directories. -Along with it a new concept `userdb` got merged that brings rich, extensible JSON user/group -records, extending the classic UNIX/glibc NSS `struct passwd`/`struct group` structures. -Both additions are added in a fully backwards compatible way, accessible through `getpwnam()`/`getgrnam()`/… (i.e. libc NSS) and PAM as -usual, meaning that for basic support no changes in the upper layers of the -stack (in particular desktop environments, such as GNOME or KDE) have to be made. -However, for better support a number of changes to desktop environments are recommended. -A few areas where that applies are discussed below. - -Before reading on, please read up on the basic concepts, specifically: - -* [Home Directories](/HOME_DIRECTORY) -* [JSON User Records](/USER_RECORD) -* [JSON Group Records](/GROUP_RECORD) -* [User/Group Record Lookup API via Varlink](/USER_GROUP_API) - -## Support for Suspending Home Directory Access during System Suspend - -One key feature of `systemd-homed` managed encrypted home directories is the -ability that access to them can be suspended automatically during system sleep, -removing any cryptographic key material from memory while doing so. -This is important in a world where most laptop users seldom shut down their computers -but most of the time just suspend them instead. -Previously, the encryption keys for the home directories remained in memory during system suspend, so that -sufficiently equipped attackers could read them from there and gain full access to the device. -By removing the key material from memory before suspend, and re-requesting it on resume this attack vector can be closed down effectively. - -Supporting this mechanism requires support in the desktop environment, since -the encryption keys (i.e. the user's login password) need to be reacquired on -system resume, from a lock screen or similar. -This lock screen must run in system context, and cannot run in the user's own context, since otherwise it -might end up accessing the home directory of the user even though access to it -is temporarily suspended and thus will hang if attempted. - -It is suggested that desktop environments that implement lock screens run them -from system context, for example by switching back to the display manager, and -only revert back to the session after re-authentication via this system lock -screen (re-authentication in this case refers to passing the user's login -credentials to the usual PAM authentication hooks). -Or in other words, when going into system suspend it is recommended that GNOME Shell switches back to -the GNOME Display Manager login screen which now should double as screen lock, -and only switches back to the shell's UI after the user re-authenticated there. - -Note that this change in behavior is a good idea in any case, and does not -create any dependencies on `systemd-homed` or systemd-specific APIs. -It's simply a change of behavior regarding use of existing APIs, not a suggested hook-up to any new APIs. - -A display manager which supports this kind of out-of-context screen lock -operation needs to inform systemd-homed about this so that systemd-homed knows -that it is safe to suspend the user's home directory on suspend. -This is done via the `suspend=` argument to the -[`pam_systemd_home`](https://www.freedesktop.org/software/systemd/man/pam_systemd_home.html) -PAM module. -A display manager should hence change its PAM stack configurationto set this parameter to on. -`systemd-homed` will not suspend home directories if there's at least one active session of the user that does not support -suspending, as communicated via this parameter. - -## User Management UIs - -The rich user/group records `userdb` and `systemd-homed` support carry various -fields of relevance to UIs that manage the local user database or parts thereof. -In particular, most of the metadata `accounts-daemon` (also see below) -supports is directly available in these JSON records. -Hence it makes sense for any user management UI to expose them directly. - -`systemd-homed` exposes APIs to add, remove and make changes to local users via -D-Bus, with full [polkit](https://www.freedesktop.org/software/polkit/docs/latest/) hook-up. -On the command line this is exposed via the `homectl` command. A graphical UI that exposes similar functionality would be -very useful, exposing the various new account settings, and in particular -providing a stream-lined UI for enrolling new-style authentication tokens such -as PKCS#11/YubiKey-style devices. -(Ideally, if the user plugs in an uninitialized YubiKey during operation it might be nice if the Desktop would -automatically ask if a key pair shall be written to it and the local account be -bound to it, `systemd-homed` provides enough YubiKey/PKCS#11 support to make -this a reality today; except that it will not take care of token -initialization). - -A strong point of `systemd-homed` is per-user resource management. -In particular disk space assignments are something that most likely should be -exposed in a user management UI. Various metadata fields are supplied allowing -exposure of disk space assignment "slider" UI. -Note however that the file system back-ends of `systemd-homed.service` have different feature sets. -Specifically, only btrfs has online file system shrinking support, ext4 only offline file -system shrinking support, and xfs no shrinking support at all (all three file -systems support online file system growing however). -This means if the LUKS back-end is used, disk space assignment cannot be instant for logged in users, unless btrfs is used. - -Note that only `systemd-homed` provides an API for modifying/creating/deleting users. -The generic `userdb` subsystem (which might have other back-ends, besides -`systemd-homed`, for example LDAP or Windows) exclusively provides a read-only interface. -(This is unlikely to change, as the other back-ends might have very -different concepts of adding or modifying users, i.e. might not even have any local concept for that at all). -This means any user management UI that intends to change (and not just view) user accounts should talk directly to -`systemd-homed` to make use of its features; there's no abstraction available -to support other back-ends under the same API. - -Unfortunately there's currently no documentation for the `systemd-homed` D-Bus API. -Consider using the `homectl` sources as guidelines for implementing a user management UI. -The JSON user/records are well documented however, see above, -and the D-Bus API provides limited introspection. - -## Relationship to `accounts-daemon` - -For a long time `accounts-daemon` has been included in Linux distributions -providing richer user accounts. -The functionality of this daemon overlaps in many areas with the functionality of `systemd-homed` or `userdb`, but there are -systematic differences, which means that `systemd-homed` cannot replace -`accounts-daemon` fully. -Most importantly: `accounts-daemon` provides "side-car" metadata for *any* type of user account, while `systemd-homed` only -provides additional metadata for the users it defines itself. -In other words: `accounts-daemon` will augment foreign accounts; `systemd-homed` cannot be used -to augment users defined elsewhere, for example in LDAP or as classic `/etc/passwd` records. - -This probably means that for the time being, a user management UI (or other UI) -that wants to support rich user records with compatibility with the status quo -ante should probably talk to both `systemd-homed` and `accounts-daemon` at the -same time, and ignore `accounts-daemon`'s records if `systemd-homed` defines them. -While I (Lennart) personally believe in the long run `systemd-homed` is -the way to go for rich user records, any UI that wants to manage and support -rich records for classic records has to support `accounts-daemon` in parallel -for the time being. - -In the short term, it might make sense to also expose the `userdb` provided -records via `accounts-daemon`, so that clients of the latter can consume them -without changes. However, I think in the long run `accounts-daemon` should -probably be removed from the general stack, hence this sounds like a temporary -solution only. - -In case you wonder, there's no automatic mechanism for converting existing -users registered in `/etc/passwd` or LDAP to users managed by `systemd-homed`. -There's documentation for doing this manually though, see -[Converting Existing Users to systemd-homed managed Users](/CONVERTING_TO_HOMED). - -## Future Additions - -JSON user/group records are extensible, hence we can easily add any additional fields desktop environments require. -For example, pattern-based authentication is likely very useful on touch-based devices, -and the user records should hence learn them natively. -Fields for other authentication mechanisms, such as fingerprint authentication should be provided as well, eventually. - -It is planned to extend the `userdb` Varlink API to support look-ups by partial -user name and real name (GECOS) data, so that log-in screens can optionally -implement simple complete-as-you-type login screens. - -It is planned to extend the `systemd-homed` D-Bus API to instantly inform clients -about hardware associated with a specific user being plugged in, to which login -screens can listen in order to initiate authentication. -Specifically, any YubiKey-like security token plugged in that is associated with a local user -record should initiate authentication for that user, making typing in of the -username unnecessary. diff --git a/docs/VIRTUAL_ROOT.md b/docs/VIRTUAL_ROOT.md deleted file mode 100644 index 9ca28e98375..00000000000 --- a/docs/VIRTUAL_ROOT.md +++ /dev/null @@ -1,689 +0,0 @@ ---- -title: Virtual Root Environment Documentation -category: Platform Documentation -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Virtual Root Environment Documentation - -## Overview - -The Virtual Root Environment is a core component of the MobileOps platform that provides isolated, containerized execution environments for mobile applications and services. It enables secure multi-tenancy, resource isolation, and consistent deployment across different environments. - -## Architecture - -### Virtual Root Components - -1. **Root Container Manager**: Orchestrates virtual root creation and lifecycle -2. **Namespace Isolation**: Provides process, network, and filesystem isolation -3. **Resource Controller**: Manages CPU, memory, and I/O resource allocation -4. **Security Framework**: Enforces security policies and access controls -5. **Storage Manager**: Handles persistent and ephemeral storage -6. **Network Bridge**: Manages virtual networking and connectivity - -### Container Technologies - -#### Chisel Runtime -Lightweight container runtime optimized for mobile workloads: -- Minimal overhead -- Fast startup times -- Efficient resource utilization -- Mobile-specific optimizations - -#### Traditional Containers -Support for standard container technologies: -- Docker containers -- Podman containers -- LXC/LXD containers -- Kubernetes pods - -## Getting Started - -### Initialize Virtual Root Environment - -```bash -# Initialize the virtual root system -./chisel_container_boot.sh prepare - -# Configure virtual networking -./network_configure.sh setup-container - -# Verify setup -./qemu_vm_boot.sh prepare -``` - -### Creating Your First Virtual Root - -```bash -# Create a basic virtual root environment -./chisel_container_boot.sh boot myapp /path/to/app/image - -# Check running containers -./chisel_container_boot.sh list - -# Monitor container status -./system_log_collector.sh monitor -``` - -## Virtual Root Configuration - -### Container Specification - -```yaml -# container-spec.yaml -apiVersion: v1 -kind: VirtualRoot -metadata: - name: mobile-app-env - namespace: production -spec: - image: mobileops/android-runtime:latest - resources: - requests: - cpu: "0.5" - memory: "1Gi" - storage: "5Gi" - limits: - cpu: "2" - memory: "4Gi" - storage: "20Gi" - security: - runAsUser: 1000 - runAsGroup: 1000 - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - networking: - mode: "bridge" - ports: - - containerPort: 8080 - hostPort: 8080 - - containerPort: 9090 - hostPort: 9090 - volumes: - - name: app-data - hostPath: /var/lib/mobileops/data - containerPath: /app/data - readOnly: false - - name: config - configMap: app-config - containerPath: /app/config - readOnly: true - environment: - - name: ENV - value: "production" - - name: DEBUG - value: "false" -``` - -### Resource Management - -#### CPU Allocation -```bash -# Set CPU limits -./chisel_container_boot.sh config --cpu-limit 2.0 --cpu-request 0.5 - -# Monitor CPU usage -./system_log_collector.sh search "cpu" -``` - -#### Memory Management -```bash -# Configure memory limits -./chisel_container_boot.sh config --memory-limit 4Gi --memory-request 1Gi - -# Monitor memory usage -./ai_core_manager.sh monitor -``` - -#### Storage Configuration -```bash -# Configure persistent storage -./asset_manager.sh add /path/to/storage storage "Container persistent storage" - -# Set up ephemeral storage -./chisel_container_boot.sh config --ephemeral-storage 10Gi -``` - -## Isolation and Security - -### Namespace Isolation - -#### Process Isolation -- Process ID (PID) namespace isolation -- Process tree separation -- Signal isolation -- Inter-process communication (IPC) isolation - -#### Network Isolation -- Network namespace separation -- Virtual network interfaces -- Firewall rules and policies -- Traffic shaping and QoS - -#### Filesystem Isolation -- Mount namespace isolation -- Root filesystem separation -- Volume mount controls -- File permission enforcement - -### Security Policies - -```bash -# Apply security policies -./toolbox_integrity_check.sh check - -# Configure security context -./chisel_container_boot.sh config \ - --security-context '{"runAsUser": 1000, "readOnlyRootFilesystem": true}' - -# Enable security scanning -./test_suite.sh security -``` - -### Access Controls - -#### Role-Based Access Control (RBAC) -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: virtual-root-user -rules: -- apiGroups: [""] - resources: ["virtualroots"] - verbs: ["get", "list", "create", "update", "delete"] -- apiGroups: [""] - resources: ["virtualroots/status"] - verbs: ["get"] -``` - -#### Security Context Constraints -```yaml -apiVersion: security.openshift.io/v1 -kind: SecurityContextConstraints -metadata: - name: mobileops-scc -allowHostDirVolumePlugin: false -allowHostIPC: false -allowHostNetwork: false -allowHostPID: false -allowHostPorts: false -allowPrivilegedContainer: false -allowedCapabilities: [] -defaultAddCapabilities: [] -requiredDropCapabilities: -- ALL -runAsUser: - type: MustRunAsRange - uidRangeMin: 1000 - uidRangeMax: 2000 -``` - -## Networking - -### Virtual Network Configuration - -#### Bridge Networking -```bash -# Set up container bridge -./network_configure.sh setup-container - -# Configure bridge networking -./chisel_container_boot.sh boot myapp /path/to/image \ - --network bridge \ - --bridge mobileops0 -``` - -#### Host Networking -```bash -# Use host networking (less secure) -./chisel_container_boot.sh boot myapp /path/to/image \ - --network host -``` - -#### Custom Networks -```bash -# Create custom network -./network_configure.sh bridge custom-net 172.20.0.1/16 - -# Use custom network -./chisel_container_boot.sh boot myapp /path/to/image \ - --network custom-net -``` - -### Service Discovery - -#### DNS Resolution -- Automatic DNS registration for containers -- Service discovery through DNS -- Load balancing integration -- Health check integration - -#### Service Mesh Integration -- Istio service mesh support -- Envoy proxy sidecar injection -- Traffic management and security -- Observability and monitoring - -## Storage Management - -### Volume Types - -#### Persistent Volumes -```bash -# Create persistent volume -./asset_manager.sh add /data/persistent storage "Persistent application data" - -# Mount persistent volume -./chisel_container_boot.sh boot myapp /path/to/image \ - --volume persistent:/app/data -``` - -#### Ephemeral Volumes -```bash -# Configure ephemeral storage -./chisel_container_boot.sh boot myapp /path/to/image \ - --ephemeral-storage 5Gi -``` - -#### Shared Volumes -```bash -# Create shared volume -./asset_manager.sh add /data/shared storage "Shared data between containers" - -# Use shared volume -./chisel_container_boot.sh boot app1 /path/to/image1 \ - --volume shared:/app/shared -./chisel_container_boot.sh boot app2 /path/to/image2 \ - --volume shared:/app/shared -``` - -### Storage Classes - -#### High-Performance Storage -- NVMe SSD storage -- Low latency access -- High IOPS capability -- Suitable for databases and AI workloads - -#### Standard Storage -- Standard SSD storage -- Balanced performance and cost -- General purpose workloads -- Default storage class - -#### Cold Storage -- Archival storage -- Cost-optimized -- Suitable for backups and logs -- Slower access times - -## Performance Optimization - -### Resource Tuning - -#### CPU Optimization -```bash -# Set CPU affinity -./chisel_container_boot.sh config --cpu-affinity "0-3" - -# Configure CPU governor -./chisel_container_boot.sh config --cpu-governor performance - -# Enable CPU scaling -./chisel_container_boot.sh config --cpu-scaling enabled -``` - -#### Memory Optimization -```bash -# Configure memory swappiness -./chisel_container_boot.sh config --memory-swappiness 10 - -# Set memory huge pages -./chisel_container_boot.sh config --memory-hugepages 2Mi - -# Configure NUMA topology -./chisel_container_boot.sh config --numa-policy preferred -``` - -#### I/O Optimization -```bash -# Set I/O scheduler -./chisel_container_boot.sh config --io-scheduler mq-deadline - -# Configure I/O priority -./chisel_container_boot.sh config --io-priority 3 - -# Enable I/O caching -./chisel_container_boot.sh config --io-cache writeback -``` - -### Performance Monitoring - -```bash -# Monitor container performance -./system_log_collector.sh monitor - -# Generate performance report -./test_suite.sh performance - -# Real-time performance metrics -./ai_core_manager.sh monitor -``` - -## High Availability and Scaling - -### Container Clustering - -#### Multi-Node Deployment -```bash -# Configure cluster nodes -./network_configure.sh setup-cluster - -# Deploy container across nodes -./chisel_container_boot.sh deploy-cluster myapp /path/to/image \ - --replicas 3 \ - --nodes node1,node2,node3 -``` - -#### Load Balancing -```bash -# Configure load balancer -./network_configure.sh setup-loadbalancer - -# Enable container load balancing -./chisel_container_boot.sh config --load-balance round-robin -``` - -### Auto-Scaling - -#### Horizontal Pod Autoscaler -```yaml -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: mobile-app-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: mobile-app - minReplicas: 2 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 -``` - -#### Vertical Pod Autoscaler -```yaml -apiVersion: autoscaling.k8s.io/v1 -kind: VerticalPodAutoscaler -metadata: - name: mobile-app-vpa -spec: - targetRef: - apiVersion: apps/v1 - kind: Deployment - name: mobile-app - updatePolicy: - updateMode: "Auto" - resourcePolicy: - containerPolicies: - - containerName: app - minAllowed: - cpu: 100m - memory: 128Mi - maxAllowed: - cpu: 4 - memory: 8Gi -``` - -## Development and Debugging - -### Development Environment Setup - -```bash -# Create development container -./chisel_container_boot.sh boot dev-env mobileops/dev-tools:latest \ - --volume $(pwd):/workspace \ - --interactive \ - --tty - -# Enable development mode -export MOBILEOPS_DEV_MODE=1 -./platform_launcher.sh start -``` - -### Debugging Tools - -#### Container Shell Access -```bash -# Execute shell in running container -./chisel_container_boot.sh exec myapp /bin/bash - -# Debug container startup -./chisel_container_boot.sh debug myapp -``` - -#### Log Analysis -```bash -# View container logs -./system_log_collector.sh search myapp - -# Stream container logs -./system_log_collector.sh monitor --follow myapp - -# Export container logs -./system_log_collector.sh export tar -``` - -#### Performance Profiling -```bash -# Profile container performance -./test_suite.sh performance myapp - -# Generate performance report -./system_log_collector.sh analyze performance -``` - -## Migration and Backup - -### Container Migration - -#### Live Migration -```bash -# Migrate running container -./chisel_container_boot.sh migrate myapp target-node - -# Checkpoint and restore -./chisel_container_boot.sh checkpoint myapp -./chisel_container_boot.sh restore myapp target-node -``` - -#### Bulk Migration -```bash -# Migrate all containers -./chisel_container_boot.sh migrate-all target-cluster - -# Migrate with validation -./chisel_container_boot.sh migrate myapp target-node --validate -``` - -### Backup and Recovery - -#### Container Backup -```bash -# Backup container state -./asset_manager.sh backup /var/lib/mobileops/containers - -# Create container snapshot -./chisel_container_boot.sh snapshot myapp -``` - -#### Disaster Recovery -```bash -# Restore from backup -./asset_manager.sh restore containers - -# Restore specific container -./chisel_container_boot.sh restore myapp /path/to/snapshot -``` - -## Security Best Practices - -1. **Principle of Least Privilege**: Grant minimal necessary permissions -2. **Regular Security Updates**: Keep base images and runtime updated -3. **Image Scanning**: Scan container images for vulnerabilities -4. **Network Segmentation**: Isolate containers using network policies -5. **Secret Management**: Use secure secret management solutions -6. **Audit Logging**: Enable comprehensive audit logging -7. **Runtime Security**: Monitor container behavior at runtime - -## Troubleshooting - -### Common Issues - -#### Container Startup Failures -```bash -# Check container logs -./system_log_collector.sh search "container startup" - -# Verify image integrity -./asset_manager.sh verify container-image - -# Check resource availability -./ai_core_manager.sh monitor -``` - -#### Network Connectivity Issues -```bash -# Test network connectivity -./network_configure.sh monitor - -# Debug network configuration -./chisel_container_boot.sh exec myapp ping target-host - -# Check network policies -./toolbox_integrity_check.sh network -``` - -#### Performance Issues -```bash -# Monitor resource usage -./system_log_collector.sh monitor - -# Analyze performance metrics -./test_suite.sh performance - -# Check for resource constraints -./ai_core_manager.sh status -``` - -### Debug Commands - -```bash -# Enable debug mode -export MOBILEOPS_DEBUG=1 - -# Verbose logging -./chisel_container_boot.sh --verbose boot myapp /path/to/image - -# Container introspection -./chisel_container_boot.sh inspect myapp - -# System resource usage -./system_log_collector.sh analyze resources -``` - -## Integration with Other Services - -### AI Core Integration -```bash -# Deploy AI workload in virtual root -./ai_core_manager.sh deploy neural-net \ - --container-runtime chisel \ - --gpu-enabled true - -# Monitor AI container -./ai_core_manager.sh monitor -``` - -### Plugin System Integration -```bash -# Install container plugin -./plugin_manager.sh install container-monitor-plugin - -# Configure plugin for virtual root -./plugin_manager.sh config container-monitor-plugin \ - --runtime chisel -``` - -## API Reference - -### Container Management API - -```bash -# REST API endpoints -GET /api/v1/containers -POST /api/v1/containers -GET /api/v1/containers/{id} -PUT /api/v1/containers/{id} -DELETE /api/v1/containers/{id} -POST /api/v1/containers/{id}/start -POST /api/v1/containers/{id}/stop -POST /api/v1/containers/{id}/restart -``` - -### Python SDK Example - -```python -from mobileops.virtualroot import ContainerManager - -# Initialize container manager -cm = ContainerManager() - -# Create container -container = cm.create_container( - name="myapp", - image="mobileops/app:latest", - resources={"cpu": "1", "memory": "2Gi"}, - volumes=[{"host": "/data", "container": "/app/data"}] -) - -# Start container -container.start() - -# Monitor container -status = container.get_status() -print(f"Container status: {status}") -``` - -## Best Practices - -1. **Resource Planning**: Plan resource allocation based on workload requirements -2. **Image Optimization**: Use minimal base images and multi-stage builds -3. **Configuration Management**: Use configuration files and environment variables -4. **Health Checks**: Implement proper health check endpoints -5. **Graceful Shutdown**: Handle shutdown signals properly -6. **Logging Strategy**: Implement structured logging -7. **Monitoring**: Set up comprehensive monitoring and alerting - -## Support and Resources - -- **Virtual Root Documentation**: [https://docs.mobileops.local/virtual-root](https://docs.mobileops.local/virtual-root) -- **Container Registry**: [https://registry.mobileops.local](https://registry.mobileops.local) -- **Community Support**: [https://community.mobileops.local/containers](https://community.mobileops.local/containers) -- **Training Materials**: [https://training.mobileops.local/virtual-root](https://training.mobileops.local/virtual-root) -- **Best Practices Guide**: [https://docs.mobileops.local/virtual-root/best-practices](https://docs.mobileops.local/virtual-root/best-practices) \ No newline at end of file diff --git a/docs/VM_INTERFACE.md b/docs/VM_INTERFACE.md deleted file mode 100644 index abe7067331d..00000000000 --- a/docs/VM_INTERFACE.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: VM Interface -category: Interfaces -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# The VM Interface - -Also consult [Writing Virtual Machine or Container -Managers](https://systemd.io/WRITING_VM_AND_CONTAINER_MANAGERS). - -systemd has a number of interfaces for interacting with virtual machine -managers, when systemd is used inside of a VM. If you work on a VM manager, -please consider supporting the following interfaces. - -1. systemd supports passing immutable binary data blobs with limited size and - restricted access to services via the `ImportCredential=`, `LoadCredential=` - and `SetCredential=` settings. These credentials may be passed into a system - via SMBIOS Type 11 vendor strings, see - [systemd(1)](https://www.freedesktop.org/software/systemd/man/latest/systemd.html) - for details. This concept may be used to flexibly configure various facets - ot the guest system. See - [systemd.system-credentials(7)](https://www.freedesktop.org/software/systemd/man/latest/systemd.system-credentials.html) - for a list of system credentials implemented by various systemd components. - -2. Readiness, information about various system properties and functionality, as - well as progress of boot may be reported by systemd to a machine manager via - the `sd_notify()` protocol via `AF_VSOCK` sockets. The address of this - socket may be configured via the `vmm.notify_socket` system credential. See - [systemd(1)](https://www.freedesktop.org/software/systemd/man/latest/systemd.html). - -3. The - [systemd-ssh-generator(8)](https://www.freedesktop.org/software/systemd/man/latest/systemd-ssh-generator.html) - functionality will automatically bind SSH login functionality to `AF_VSOCK` - port 22, if the system runs in a VM. - -4. If not initialized yet the system's - [machine-id(5)](https://www.freedesktop.org/software/systemd/man/latest/machine-id.html) - is automatically set to the SMBIOS product UUID if available and invocation - in an VM environment is detected. - -5. The - [`systemd-boot(7)`](https://www.freedesktop.org/software/systemd/man/latest/systemd-boot.html) - and - [`systemd-stub(7)`](https://www.freedesktop.org/software/systemd/man/latest/systemd-stub.html) - components support two SMBIOS Type 11 vendor strings that may be used to - extend the kernel command line of booted Linux environments: - `io.systemd.stub.kernel-cmdline-extra=` and - `io.systemd.boot.kernel-cmdline-extra=`. - -Also see -[smbios-type-11(7)](https://www.freedesktop.org/software/systemd/man/latest/smbios-type-11.html) -for a list of supported SMBIOS Type 11 vendor strings. diff --git a/docs/WRITING_DESKTOP_ENVIRONMENTS.md b/docs/WRITING_DESKTOP_ENVIRONMENTS.md deleted file mode 100644 index 774308df782..00000000000 --- a/docs/WRITING_DESKTOP_ENVIRONMENTS.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Writing Desktop Environments -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Writing Desktop Environments - -_Or: how to hook up your favorite desktop environment with logind_ - -systemd's logind service obsoletes ConsoleKit which was previously widely used on Linux distributions. -This provides a number of new features, but also requires updating of the Desktop Environment running on it, in a few ways. - -This document should be read together with [Writing Display Managers](/WRITING_DISPLAY_MANAGERS) which focuses on the porting work necessary for display managers. - -If required it is possible to implement ConsoleKit and systemd-logind support in the same desktop environment code, detecting at runtime which interface is needed. -The [sd_booted()](http://www.freedesktop.org/software/systemd/man/sd_booted.html) call may be used to determine at runtime whether systemd is used. - -To a certain level ConsoleKit and systemd-logind may be used side-by-side, but a number of features are not available if ConsoleKit is used. - -Please have a look at the [Bus API of logind](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.login1.html) and the C API as documented in [sd-login(7)](http://www.freedesktop.org/software/systemd/man/sd-login.html). (Also see below) - -Here are the suggested changes: - -- Your session manager should listen to "Lock" and "Unlock" messages that are emitted from the session object logind exposes for your DE session, on the system bus. - If "Lock" is received the screen lock should be activated, if "Unlock" is received it should be deactivated. - This can easily be tested with "loginctl lock-sessions". - See the [Bus API of logind](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.login1.html) for further details. -- Whenever the session gets idle the DE should invoke the SetIdleHint(True) call on the respective session object on the session bus. - This is necessary for the system to implement auto-suspend when all sessions are idle. - If the session gets used again it should call SetIdleHint(False). - A session should be considered idle if it didn't receive user input (mouse movements, keyboard) in a while. - See the [Bus API of logind](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.login1.html) for further details. -- To reboot/power-off/suspend/hibernate the machine from the DE use logind's bus calls Reboot(), PowerOff(), Suspend(), Hibernate(), HybridSleep(). - For further details see [Bus API of logind](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.login1.html). -- If your session manager handles the special power, suspend, hibernate hardware keys or the laptop lid switch on its own it is welcome to do so, - but needs to disable logind's built-in handling of these events. - Take one or more of the _handle-power-key_, _handle-suspend-key_, _handle-hibernate-key_, _handle-lid-switch_ inhibitor locks for that. - See [Inhibitor Locks](/INHIBITOR_LOCKS) for further details on this. -- Before rebooting/powering-off/suspending/hibernating and when the operation is triggered by the user by clicking on some UI elements - (or suchlike) it is recommended to show the list of currently active inhibitors for the operation, and ask the user to acknowledge the operation. - Note that PK often allows the user to execute the operation ignoring the inhibitors. - Use logind's ListInhibitors() call to get a list of these inhibitors. See [Inhibitor Locks](/INHIBITOR_LOCKS) for further details on this. -- If your DE contains a process viewer of some kind ("system monitor") it's a good idea to show session, service and seat information for each process. - Use sd_pid_get_session(), sd_pid_get_unit(), sd_session_get_seat() to determine these. - For details see [sd-login(7)](http://www.freedesktop.org/software/systemd/man/sd-login.html). - -And that's all! Thank you! diff --git a/docs/WRITING_DISPLAY_MANAGERS.md b/docs/WRITING_DISPLAY_MANAGERS.md deleted file mode 100644 index 1fe70d044a7..00000000000 --- a/docs/WRITING_DISPLAY_MANAGERS.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: Writing Display Managers -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Writing Display Managers - -_Or: How to hook up your favorite X11 display manager with systemd_ - -systemd's logind service obsoletes ConsoleKit which was previously widely used on Linux distributions. -For X11 display managers the switch to logind requires a minimal amount of porting, however brings a couple of new features: -true automatic multi-seat support, proper tracking of session processes, (optional) automatic killing of user processes on logout, a synchronous low-level C API and much simplification. - -This document should be read together with [Writing Desktop Environments](/WRITING_DESKTOP_ENVIRONMENTS) which focuses on the porting work necessary for desktop environments. - -If required it is possible to implement ConsoleKit and systemd-logind support in the same display manager, detecting at runtime which interface is needed. -The [sd_booted()](http://www.freedesktop.org/software/systemd/man/sd_booted.html) call may be used to determine at runtime whether systemd is used. - -To a certain level ConsoleKit and systemd-logind may be used side-by-side, but a number of features are not available if ConsoleKit is used, for example automatic multi-seat support. - -Please have a look at the [Bus API of logind](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.login1.html) and the C API as documented in [sd-login(7)](http://www.freedesktop.org/software/systemd/man/sd-login.html). -(Also see below) - -Minimal porting (without multi-seat) requires the following: - -1. Remove/disable all code responsible for registering your service with ConsoleKit. -2. Make sure to register your greeter session via the PAM session stack, and make sure the PAM session modules include pam_systemd. - Also, make sure to set the session class to "greeter." This may be done by setting the environment variable XDG_SESSION_CLASS to "greeter" with pam_misc_setenv() or setting the "class=greeter" option in the pam_systemd module, in order to allow applications to filter out greeter sessions from normal login sessions. -3. Make sure to register your logged in session via the PAM session stack as well, also including pam_systemd in it. -4. Optionally, use pam_misc_setenv() to set the environment variables XDG_SEAT and XDG_VTNR. - The former should contain "seat0", the latter the VT number your session runs on. pam_systemd can determine these values automatically but it's nice to pass these variables anyway. -In summary: porting a display manager from ConsoleKit to systemd primarily means removing code, not necessarily adding any new code. Here, a cheers to simplicity! - -1. Subscribe to seats showing up and going away, via the systemd-logind D-Bus interface's SeatAdded and SeatRemoved signals. - Take possession of each seat by spawning your greeter on it. - However, do so exclusively for seats where the boolean CanGraphical property is true. - Note that there are seats that cannot do graphical, and there are seats that are text-only first, and gain graphical support later on. - Most prominently this is actually seat0 which comes up in text mode, and where the graphics driver is then loaded and probed during boot. - This means display managers must watch PropertyChanged events on all seats, to see if they gain (or lose) the CanGraphical field. -2. Use ListSeats() on the D-Bus interface to acquire a list of already available seats and also take possession of them. -3. For each seat you spawn a greeter/user session on use the XDG_SEAT and XDG_VTNR PAM environment variables to inform pam_systemd about the seat name, resp. - VT number you start them on. Note that only the special seat "seat0" actually knows kernel VTs, so you shouldn't pass the VT number on any but the main seat, since it doesn't make any sense there. -4. Pass the seat name to the X server you start via the -seat parameter. -5. At this time X interprets the -seat parameter natively only for input devices, not for graphics devices. - To work around this limitation we provide a tiny wrapper /lib/systemd/systemd-multi-seat-x which emulates the enumeration for graphics devices too. - This wrapper will eventually go away, as soon as X learns udev-based graphics device enumeration natively, instead of the current PCI based one. - Hence it is a good idea to fall back to the real X when this wrapper is not found. - You may use this wrapper exactly like the real X server, and internally it will just exec() it after putting together a minimal multi-seat configuration. - And that's already it. - -While most information about seats, sessions and users is available on systemd-logind's D-Bus interface, this is not the only API. -The synchronous [sd-login(7)](http://www.freedesktop.org/software/systemd/man/sd-login.html) C interface is often easier to use and much faster too. -In fact it is possible to implement the scheme above entirely without D-Bus relying only on this API. -Note however, that this C API is purely passive, and if you want to execute an actually state changing operation you need to use the bus interface (for example, to switch sessions, or to kill sessions and suchlike). -Also have a look at the [logind Bus API](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.login1.html). diff --git a/docs/WRITING_NETWORK_CONFIGURATION_MANAGERS.md b/docs/WRITING_NETWORK_CONFIGURATION_MANAGERS.md deleted file mode 100644 index 722076504e6..00000000000 --- a/docs/WRITING_NETWORK_CONFIGURATION_MANAGERS.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -title: Writing Network Configuration Managers -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Writing Network Configuration Managers - -_Or: How to hook up your favourite network configuration manager's DNS logic with `systemd-resolved`_ - -_(This is a longer explanation how to use some parts of `systemd-resolved` bus API. If you are just looking for an API reference, consult the [bus API documentation](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.resolve1.html) instead.)_ - -Since systemd 229 `systemd-resolved` offers a powerful bus API that may be used by network configuration managers (e.g. NetworkManager, connman, …, but also lower level DHCP, VPN or PPP daemons managing specific interfaces) to pass DNS server and DNSSEC configuration directly to `systemd-resolved`. -Note that `systemd-resolved` also reads the DNS configuration data in `/etc/resolv.conf`, for compatibility. However, by passing the DNS configuration directly to `systemd-resolved` via the bus a couple of benefits are available: - -1. `systemd-resolved` maintains DNS configuration per-interface, instead of simply system-wide, - and is capable of sending DNS requests to servers on multiple different network interfaces simultaneously, returning the first positive response - (or if all fail, the last negative one). - This allows effective "merging" of DNS views on different interfaces, which makes private DNS zones on multi-homed hosts a lot nicer to use. - For example, if you are connected to a LAN and a VPN, and both have private DNS zones, then you will be able to resolve both, as long as they don't clash in names. - By using the bus API to configure DNS settings, the per-interface configuration is opened up. -2. Per-link configuration of DNSSEC is available. This is particularly interesting for network configuration managers that implement captive portal detection: - as long as a verified connection to the Internet is not found DNSSEC should be turned off - (as some captive portal systems alter the DNS in order to redirect clients to their internal pages). -3. Per-link configuration of LLMNR and MulticastDNS is available. -4. In contrast to changes to `/etc/resolv.conf` all changes made via the bus take effect immediately for all future lookups. -5. Statistical data about executed DNS transactions is available, as well as information about whether DNSSEC is supported on the chosen DNS server. - -Note that `systemd-networkd` is already hooked up with `systemd-resolved`, exposing this functionality in full. - -## Suggested Mode of Operation - -Whenever a network configuration manager sets up an interface for operation, it should pass the DNS configuration information for the interface to `systemd-resolved`. -It's recommended to do that after the Linux network interface index ("ifindex") has been allocated, but before the interface has been upped (i.e. `IFF_UP` turned on). -That way, `systemd-resolved` will be able to use the configuration the moment the network interface is available. -(Note that `systemd-resolved` watches the kernel interfaces come and go, and will make use of them as soon as they are suitable to be used, which among other factors requires `IFF_UP` to be set). -That said it is OK to change DNS configuration dynamically any time: simply pass the new data to resolved, and it is happy to use it. - -In order to pass the DNS configuration information to resolved, use the following methods of the `org.freedesktop.resolve1.Manager` interface of the `/org/freedesktop/resolve1` object, on the `org.freedesktop.resolve1` service: - -1. To set the DNS server IP addresses for a network interface, use `SetLinkDNS()` -2. To set DNS search and routing domains for a network interface, use `SetLinkDomains()` -3. To configure the DNSSEC mode for a network interface, use `SetLinkDNSSEC()` -4. To configure DNSSEC Negative Trust Anchors (NTAs, i.e. domains for which not to do DNSSEC validation), use `SetLinkDNSSECNegativeTrustAnchors()` -5. To configure the LLMNR and MulticastDNS mode, use `SetLinkLLMNR()` and `SetLinkMulticastDNS()` - -For details about these calls see the [full resolved bus API documentation](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.resolve1.html). - -The calls should be pretty obvious to use: they simply take an interface index and the parameters to set. -IP addresses are encoded as an address family specifier (an integer, that takes the usual `AF_INET` and `AF_INET6` constants), followed by a 4 or 16 byte array with the address in network byte order. - -`systemd-resolved` distinguishes between "search" and "routing" domains. -Routing domains are used to route DNS requests of specific domains to particular interfaces. -i.e. requests for a hostname `foo.bar.com` will be routed to any interface that has `bar.com` as routing domain. -The same routing domain may be defined on multiple interfaces, in which case the request is routed to all of them in parallel. -Resolver requests for hostnames that do not end in any defined routing domain of any interface will be routed to all suitable interfaces. -Search domains work like routing domain, but are also used to qualify single-label domain names. -They hence are identical to the traditional search domain logic on UNIX. -The `SetLinkDomains()` call may used to define both search and routing domains. - -The most basic support of `systemd-resolved` in a network configuration manager would be to simply invoke `SetLinkDNS()` and `SetLinkDomains()` for the specific interface index with the data traditionally written to `/etc/resolv.conf`. -More advanced integration could mean the network configuration manager also makes the DNSSEC mode, the DNSSEC NTAs and the LLMNR/MulticastDNS modes available for configuration. - -It is strongly recommended for network configuration managers that implement captive portal detection to turn off DNSSEC validation during the detection phase, so that captive portals that modify DNS do not result in all DNSSEC look-ups to fail. - -If a network configuration manager wants to reset specific settings to the defaults (such as the DNSSEC, LLMNR or MulticastDNS mode), it may simply call the function with an empty argument. -To reset all per-link changes it made it may call `RevertLink()`. - -To read back the various settings made, use `GetLink()` to get a `org.freedesktop.resolve1.Link` object for a specific network interface. -It exposes the current settings in its bus properties. -See the [full bus API documentation](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.resolve1.html) for details on this. - -In order to translate a network interface name to an interface index, use the usual glibc `if_nametoindex()` call. - -If the network configuration UI shall expose information about whether the selected DNS server supports DNSSEC, check the `DNSSECSupported` on the link object. - -Note that it is fully OK if multiple different daemons push DNS configuration data into `systemd-resolved` as long as they do this only for the network interfaces they own and manage. - -## Handling of `/etc/resolv.conf` - -`systemd-resolved` receives DNS configuration from a number of sources, via the bus, as well as directly from `systemd-networkd` or user configuration. -It uses this data to write a file that is compatible with the traditional Linux `/etc/resolv.conf` file. -This file is stored in `/run/systemd/resolve/resolv.conf`. It is recommended to symlink `/etc/resolv.conf` to this file, in order to provide compatibility with programs reading the file directly and not going via the NSS and thus `systemd-resolved`. - -For network configuration managers it is recommended to rely on this resolved-provided mechanism to update `resolv.conf`. -Specifically, the network configuration manager should stop modifying `/etc/resolv.conf` directly if it notices it being a symlink to `/run/systemd/resolve/resolv.conf`. - -If a system configuration manager desires to be compatible both with systems that use `systemd-resolved` and those which do not, it is recommended to first push any discovered DNS configuration into `systemd-resolved`, and deal gracefully with `systemd-resolved` not being available on the bus. -If `/etc/resolv.conf` is a not a symlink to `/run/systemd/resolve/resolv.conf` the manager may then proceed and also update `/etc/resolv.conf`. -With this mode of operation optimal compatibility is provided, as `systemd-resolved` is used for `/etc/resolv.conf` management when this is configured, but transparent compatibility with non-`systemd-resolved` systems is maintained. -Note that `systemd-resolved` is part of systemd, and hence likely to be pretty universally available on Linux systems soon. - -By allowing `systemd-resolved` to manage `/etc/resolv.conf` ownership issues regarding different programs overwriting each other's DNS configuration are effectively removed. diff --git a/docs/WRITING_RESOLVER_CLIENTS.md b/docs/WRITING_RESOLVER_CLIENTS.md deleted file mode 100644 index 93c51c53880..00000000000 --- a/docs/WRITING_RESOLVER_CLIENTS.md +++ /dev/null @@ -1,306 +0,0 @@ ---- -title: Writing Resolver Clients -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Writing Resolver Clients - -_Or: How to look up hostnames and arbitrary DNS Resource Records via_ `systemd-resolved` _'s bus APIs_ - -_(This is a longer explanation how to use some parts of_ `systemd-resolved` _bus API. If you are just looking for an API reference, consult the [bus API documentation](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.resolve1.html) instead.)_ - -_`systemd-resolved`_ provides a set of APIs on the bus for resolving DNS resource records. These are: - -1. _ResolveHostname()_ for resolving hostnames to acquire their IP addresses -2. _ResolveAddress()_ for the reverse operation: acquire the hostname for an IP address -3. _ResolveService()_ for resolving a DNS-SD or SRV service -4. _ResolveRecord()_ for resolving arbitrary resource records. - -Below you'll find examples for two of these calls, to show how to use them. -Note that glibc offers similar (and more portable) calls in _getaddrinfo()_, _getnameinfo()_ and _res\_query()_. -Of these _getaddrinfo()_ and _getnameinfo()_ are directed to the calls above via the _nss-resolve_ NSS module, but _req\_query()_ is not. -There are a number of reasons why it might be preferable to invoke `systemd-resolved`'s bus calls rather than the glibc APIs: - -1. Bus APIs are naturally asynchronous, which the glibc APIs generally are not. -2. The bus calls above pass back substantially more information about the resolved data, including where and how the data was found - (i.e. which protocol was used: DNS, LLMNR, MulticastDNS, and on which network interface), and most importantly, whether the data could be authenticated via DNSSEC. - This in particular makes these APIs useful for retrieving certificate data from the DNS, in order to implement DANE, SSHFP, OPENGPGKEY and IPSECKEY clients. -3. _ResolveService()_ knows no counterpart in glibc, and has the benefit of being a single call that collects all data necessary to connect to a DNS-SD or pure SRV service in one step. -4. _ResolveRecord()_ in contrast to _res\_query()_ supports LLMNR and MulticastDNS as protocols on top of DNS, and makes use of `systemd-resolved`'s local DNS record cache. - The processing of the request is done in the sandboxed `systemd-resolved` process rather than in the local process, and all packets are pre-validated. - Because this relies on `systemd-resolved` the per-interface DNS zone handling is supported. - -Of course, by using `systemd-resolved` you lose some portability, but this could be handled via an automatic fallback to the glibc counterparts. - -Note that the various resolver calls provided by `systemd-resolved` will consult `/etc/hosts` and synthesize resource records for these entries in order to ensure that this file is honoured fully. - -The examples below use the _sd-bus_ D-Bus client implementation, which is part of _libsystemd_. -Any other D-Bus library, including the original _libdbus_ or _GDBus_ may be used too. - -## Resolving a Hostname - -To resolve a hostname use the _ResolveHostname()_ call. For details on the function parameters see the [bus API documentation](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.resolve1.html). - -This example specifies `AF_UNSPEC` as address family for the requested address. -This means both an _AF\_INET_ (A) and an _AF\_INET6_ (AAAA) record is looked for, depending on whether the local system has configured IPv4 and/or IPv6 connectivity. -It is generally recommended to request `AF_UNSPEC` addresses for best compatibility with both protocols, in particular on dual-stack systems. - -The example specifies a network interface index of "0", i.e. does not specify any at all, so that the request may be done on any. -Note that the interface index is primarily relevant for LLMNR and MulticastDNS lookups, which distinguish different scopes for each network interface index. - -This examples makes no use of either the input flags parameter, nor the output flags parameter. -See the _ResolveRecord()_ example below for information how to make use of the _SD\_RESOLVED\_AUTHENTICATED_ bit in the returned flags parameter. - -```c -#include -#include -#include -#include -#include -#include - -int main(int argc, char*argv[]) { - sd_bus_error error = SD_BUS_ERROR_NULL; - sd_bus_message *reply = NULL; - const char *canonical; - sd_bus *bus = NULL; - uint64_t flags; - int r; - - r = sd_bus_open_system(&bus); - if (r < 0) { - fprintf(stderr, "Failed to open system bus: %s\n", strerror(-r)); - goto finish; - } - - r = sd_bus_call_method(bus, - "org.freedesktop.resolve1", - "/org/freedesktop/resolve1", - "org.freedesktop.resolve1.Manager", - "ResolveHostname", - &error, - &reply, - "isit", - 0, /* Network interface index where to look (0 means any) */ - argc >= 2 ? argv[1] : "www.github.com", /* Hostname */ - AF_UNSPEC, /* Which address family to look for */ - UINT64_C(0)); /* Input flags parameter */ - if (r < 0) { - fprintf(stderr, "Failed to resolve hostnme: %s\n", error.message); - sd_bus_error_free(&error); - goto finish; - } - - r = sd_bus_message_enter_container(reply, 'a', "(iiay)"); - if (r < 0) - goto parse_failure; - - for (;;) { - char buf[INET6_ADDRSTRLEN]; - int ifindex, family; - const void *data; - size_t length; - - r = sd_bus_message_enter_container(reply, 'r', "iiay"); - if (r < 0) - goto parse_failure; - if (r == 0) /* Reached end of array */ - break; - r = sd_bus_message_read(reply, "ii", &ifindex, &family); - if (r < 0) - goto parse_failure; - r = sd_bus_message_read_array(reply, 'y', &data, &length); - if (r < 0) - goto parse_failure; - r = sd_bus_message_exit_container(reply); - if (r < 0) - goto parse_failure; - - printf("Found IP address %s on interface %i.\n", inet_ntop(family, data, buf, sizeof(buf)), ifindex); - } - - r = sd_bus_message_exit_container(reply); - if (r < 0) - goto parse_failure; - r = sd_bus_message_read(reply, "st", &canonical, &flags); - if (r < 0) - goto parse_failure; - - printf("Canonical name is %s\n", canonical); - goto finish; - -parse_failure: - fprintf(stderr, "Parse failure: %s\n", strerror(-r)); - -finish: - sd_bus_message_unref(reply); - sd_bus_flush_close_unref(bus); - return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; -} -``` - -Compile this with a command line like the following (under the assumption you save the sources above as `addrtest.c`): - -``` -gcc addrtest.c -o addrtest -Wall `pkg-config --cflags --libs libsystemd` -``` - -## Resolving an Arbitrary DNS Resource Record - -Use `ResolveRecord()` in order to resolve arbitrary resource records. The call will return the binary RRset data. -This calls is useful to acquire resource records for which no high-level calls such as ResolveHostname(), ResolveAddress() and ResolveService() exist. -In particular RRs such as MX, SSHFP, TLSA, CERT, OPENPGPKEY or IPSECKEY may be requested via this API. - -This example also shows how to determine whether the acquired data has been authenticated via DNSSEC (or another means) by checking the `SD_RESOLVED_AUTHENTICATED` bit in the -returned `flags` parameter. - -This example contains a simple MX record parser. -Note that the data comes pre-validated from `systemd-resolved`, hence we allow the example to parse the record slightly sloppily, to keep the example brief. -For details on the MX RR binary format, see [RFC 1035](https://www.rfc-editor.org/rfc/rfc1035.txt). - -For details on the function parameters see the [bus API documentation](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.resolve1.html). - -```c -#include -#include -#include -#include -#include -#include -#include - -#define DNS_CLASS_IN 1U -#define DNS_TYPE_MX 15U - -#define SD_RESOLVED_AUTHENTICATED (UINT64_C(1) << 9) - -static const uint8_t* print_name(const uint8_t* p) { - bool dot = false; - for (;;) { - if (*p == 0) - return p + 1; - if (dot) - putchar('.'); - else - dot = true; - printf("%.*s", (int) *p, (const char*) p + 1); - p += *p + 1; - } -} - -static void process_mx(const void *rr, size_t sz) { - uint16_t class, type, rdlength, preference; - const uint8_t *p = rr; - uint32_t ttl; - - fputs("Found MX: ", stdout); - p = print_name(p); - - memcpy(&type, p, sizeof(uint16_t)); - p += sizeof(uint16_t); - memcpy(&class, p, sizeof(uint16_t)); - p += sizeof(uint16_t); - memcpy(&ttl, p, sizeof(uint32_t)); - p += sizeof(uint32_t); - memcpy(&rdlength, p, sizeof(uint16_t)); - p += sizeof(uint16_t); - memcpy(&preference, p, sizeof(uint16_t)); - p += sizeof(uint16_t); - - assert(be16toh(type) == DNS_TYPE_MX); - assert(be16toh(class) == DNS_CLASS_IN); - printf(" preference=%u ", be16toh(preference)); - - p = print_name(p); - putchar('\n'); - - assert(p == (const uint8_t*) rr + sz); -} - -int main(int argc, char*argv[]) { - sd_bus_error error = SD_BUS_ERROR_NULL; - sd_bus_message *reply = NULL; - sd_bus *bus = NULL; - uint64_t flags; - int r; - - r = sd_bus_open_system(&bus); - if (r < 0) { - fprintf(stderr, "Failed to open system bus: %s\n", strerror(-r)); - goto finish; - } - - r = sd_bus_call_method(bus, - "org.freedesktop.resolve1", - "/org/freedesktop/resolve1", - "org.freedesktop.resolve1.Manager", - "ResolveRecord", - &error, - &reply, - "isqqt", - 0, /* Network interface index where to look (0 means any) */ - argc >= 2 ? argv[1] : "gmail.com", /* Domain name */ - DNS_CLASS_IN, /* DNS RR class */ - DNS_TYPE_MX, /* DNS RR type */ - UINT64_C(0)); /* Input flags parameter */ - if (r < 0) { - fprintf(stderr, "Failed to resolve record: %s\n", error.message); - sd_bus_error_free(&error); - goto finish; - } - - r = sd_bus_message_enter_container(reply, 'a', "(iqqay)"); - if (r < 0) - goto parse_failure; - - for (;;) { - uint16_t class, type; - const void *data; - size_t length; - int ifindex; - - r = sd_bus_message_enter_container(reply, 'r', "iqqay"); - if (r < 0) - goto parse_failure; - if (r == 0) /* Reached end of array */ - break; - r = sd_bus_message_read(reply, "iqq", &ifindex, &class, &type); - if (r < 0) - goto parse_failure; - r = sd_bus_message_read_array(reply, 'y', &data, &length); - if (r < 0) - goto parse_failure; - r = sd_bus_message_exit_container(reply); - if (r < 0) - goto parse_failure; - - process_mx(data, length); - } - - r = sd_bus_message_exit_container(reply); - if (r < 0) - goto parse_failure; - r = sd_bus_message_read(reply, "t", &flags); - if (r < 0) - goto parse_failure; - - printf("Response is authenticated: %s\n", flags & SD_RESOLVED_AUTHENTICATED ? "yes" : "no"); - goto finish; - -parse_failure: - fprintf(stderr, "Parse failure: %s\n", strerror(-r)); - -finish: - sd_bus_message_unref(reply); - sd_bus_flush_close_unref(bus); - return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS; - } -``` - -Compile this with a command line like the following (under the assumption you save the sources above as `rrtest.c`): - -``` -gcc rrtest.c -o rrtest -Wall `pkg-config --cflags --libs libsystemd` -``` diff --git a/docs/WRITING_VM_AND_CONTAINER_MANAGERS.md b/docs/WRITING_VM_AND_CONTAINER_MANAGERS.md deleted file mode 100644 index 724d3d6dafb..00000000000 --- a/docs/WRITING_VM_AND_CONTAINER_MANAGERS.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: Writing VM and Container Managers -category: Documentation for Developers -layout: default -SPDX-License-Identifier: LGPL-2.1-or-later ---- - -# Writing VM and Container Managers - -_Or: How to hook up your favorite VM or container manager with systemd_ - -Nomenclature: a _Virtual Machine_ shall refer to a system running on virtualized hardware consisting of a full OS with its own kernel. -A _Container_ shall refer to a system running on the same shared kernel of the host, but running a mostly complete OS with its own init system. -Both kinds of virtualized systems shall collectively be called "machines". - -systemd provides a number of integration points with virtual machine and container managers, such as libvirt, LXC or systemd-nspawn. -On one hand there are integration points of the VM/container manager towards the host OS it is running on, and on the other there integration points for container managers towards the guest OS it is managing. - -Note that this document does not cover lightweight containers for the purpose -of application sandboxes, i.e. containers that do _not_ run a init system of -their own. - -## Host OS Integration - -All virtual machines and containers should be registered with the [machined](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.machine1) mini service that is part of systemd. This provides integration into the core OS at various points. For example, tools like ps, cgls, gnome-system-manager use this registration information to show machine information for running processes, as each of the VM's/container's processes can reliably attributed to a registered machine. -The various systemd tools (like systemctl, journalctl, loginctl, systemd-run, ...) all support a -M switch that operates on machines registered with machined. -"machinectl" may be used to execute operations on any such machine. -When a machine is registered via machined its processes will automatically be placed in a systemd scope unit (that is located in the machines.slice slice) and thus appear in "systemctl" and similar commands. -The scope unit name is based on the machine meta information passed to machined at registration. - -For more details on the APIs provided by machine consult [the bus API interface documentation](https://www.freedesktop.org/software/systemd/man/latest/org.freedesktop.machine1). - -## Guest OS Integration - -As container virtualization is much less comprehensive, and the guest is less isolated from the host, there are a number of interfaces defined how the container manager can set up the environment for systemd running inside a container. These Interfaces are documented in [Container Interface of systemd](/CONTAINER_INTERFACE). - -VM virtualization is more comprehensive and fewer integration APIs are available. -In fact there's only one: a VM manager may initialize the SMBIOS DMI field "Product UUUID" to a UUID uniquely identifying this virtual machine instance. -This is read in the guest via `/sys/class/dmi/id/product_uuid`, and used as configuration source for `/etc/machine-id` if in the guest, if that file is not initialized yet. -Note that this is currently only supported for kvm hosts, but may be extended to other managers as well. diff --git a/docs/index.md b/docs/index.md index 3c05c935504..4f45a2cc8c5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,96 +3,54 @@ layout: default SPDX-License-Identifier: LGPL-2.1-or-later --- -# System and Service Manager +# FileSystemds Mobile Platform -systemd is a suite of basic building blocks for a Linux system. It provides a system and service manager that runs as PID 1 and starts the rest of the system. +FileSystemds is a modular, agent-driven orchestration system designed for modern mobile/cloud-first environments. Built from highly modified foundations, it's evolving into a next-generation platform for mobile operations. -systemd provides aggressive parallelization capabilities, uses socket and D-Bus activation for starting services, offers on-demand starting of daemons, keeps track of processes using Linux control groups, maintains mount and automount points, and implements an elaborate transactional dependency-based service control logic. systemd supports SysV and LSB init scripts and works as a replacement for sysvinit. +## Key Features -Other parts include a logging daemon, utilities to control basic system configuration like the hostname, date, locale, maintain a list of logged-in users and running containers and virtual machines, system accounts, runtime directories and settings, and daemons to manage simple network configuration, network time synchronization, log forwarding, and name resolution. +- **Modular Architecture**: Replaceable components instead of monolithic design +- **Agent-Driven Workflows**: Event-driven, API-first automation for both human and non-human contributors +- **Mobile/Cloud/Edge Ready**: Cross-platform, stateless design optimized for modern environments +- **Pointer-First Artifact Management**: Secure, audited handling of large assets and models +- **Observable & Testable**: Structured logging, metrics, and comprehensive testing ---- +## Getting Started + +For complete installation and usage instructions, see the [Mobile Platform Documentation](../README_MOBILE.md). + +## Documentation Categories {% assign by_category = site.pages | group_by:"category" %} {% assign extra_pages = site.data.extra_pages | group_by:"category" %} {% assign merged = by_category | concat: extra_pages | sort:"name" %} {% for pair in merged %} - {% if pair.name != "" %} -## {{ pair.name }} -{% assign sorted = pair.items | sort:"title" %}{% for page in sorted %} -* [{{ page.title }}]({{ page.url | relative_url }}){% endfor %} + {% assign category = pair.name %} + {% if category != "" %} + +### {{ category }} + + {% assign sorted_pages = pair.items | sort: "title" %} + {% for page in sorted_pages %} + {% if page.title and page.title != "" %} +* [{{ page.title }}]({{ page.url | relative_url }}) + {% endif %} + {% endfor %} {% endif %} {% endfor %} -## See also +## Architecture Overview -* [Introductory blog story](https://0pointer.de/blog/projects/systemd.html) -* [Three](https://0pointer.de/blog/projects/systemd-update.html) [status](https://0pointer.de/blog/projects/systemd-update-2.html) [updates](https://0pointer.de/blog/projects/systemd-update-3.html) -* The [Wikipedia article](https://en.wikipedia.org/wiki/systemd) +FileSystemds represents a transition from legacy personal computing software to a modern, modular platform. The architecture emphasizes: ---- +- **Modular Services**: Each major feature is a replaceable module +- **API-First Design**: All functionality is accessible via clean APIs +- **Event-Driven Operations**: Reactive, autonomous workflows +- **Mobile-First Principles**: Optimized for mobile and edge computing environments + +For detailed architectural information, see the [Architecture Documentation](ARCHITECTURE.md). + +## Contributing -
-Welcome to Fedora 20 (Heisenbug)!
-
-[  OK  ] Reached target Remote File Systems.
-[  OK  ] Listening on Delayed Shutdown Socket.
-[  OK  ] Listening on /dev/initctl Compatibility Named Pipe.
-[  OK  ] Reached target Paths.
-[  OK  ] Reached target Encrypted Volumes.
-[  OK  ] Listening on Journal Socket.
-         Mounting Huge Pages File System...
-         Mounting POSIX Message Queue File System...
-         Mounting Debug File System...
-         Starting Journal Service...
-[  OK  ] Started Journal Service.
-         Mounting Configuration File System...
-         Mounting FUSE Control File System...
-[  OK  ] Created slice Root Slice.
-[  OK  ] Created slice User and Session Slice.
-[  OK  ] Created slice System Slice.
-[  OK  ] Reached target Slices.
-[  OK  ] Reached target Swap.
-         Mounting Temporary Directory...
-[  OK  ] Reached target Local File Systems (Pre).
-         Starting Load Random Seed...
-         Starting Load/Save Random Seed...
-[  OK  ] Mounted Huge Pages File System.
-[  OK  ] Mounted POSIX Message Queue File System.
-[  OK  ] Mounted Debug File System.
-[  OK  ] Mounted Configuration File System.
-[  OK  ] Mounted FUSE Control File System.
-[  OK  ] Mounted Temporary Directory.
-[  OK  ] Started Load Random Seed.
-[  OK  ] Started Load/Save Random Seed.
-[  OK  ] Reached target Local File Systems.
-         Starting Recreate Volatile Files and Directories...
-         Starting Trigger Flushing of Journal to Persistent Storage...
-[  OK  ] Started Recreate Volatile Files and Directories.
-         Starting Record System Reboot/Shutdown in UTMP...
-[  OK  ] Started Trigger Flushing of Journal to Persistent Storage.
-[  OK  ] Started Record System Reboot/Shutdown in UTMP.
-[  OK  ] Reached target System Initialization.
-[  OK  ] Reached target Timers.
-[  OK  ] Listening on D-Bus System Message Bus Socket.
-[  OK  ] Reached target Sockets.
-[  OK  ] Reached target Basic System.
-         Starting Permit User Sessions...
-         Starting D-Bus System Message Bus...
-[  OK  ] Started D-Bus System Message Bus.
-         Starting Login Service...
-         Starting Cleanup of Temporary Directories...
-[  OK  ] Started Permit User Sessions.
-[  OK  ] Started Cleanup of Temporary Directories.
-         Starting Console Getty...
-[  OK  ] Started Console Getty.
-[  OK  ] Reached target Login Prompts.
-[  OK  ] Started Login Service.
-[  OK  ] Reached target Multi-User System.
-
-Fedora release 20 (Heisenbug)
-Kernel 3.9.2-200.fc18.x86_64 on an x86_64 (console)
-
-fedora login:
-
+We welcome contributions that advance our modular, agent-driven vision. See our [Contributing Guidelines](CONTRIBUTING.md) for details on how to get involved. \ No newline at end of file diff --git a/scripts/ai_core_manager.sh b/scripts/ai_core_manager.sh index e74f8c1f032..175ac1d4093 100755 --- a/scripts/ai_core_manager.sh +++ b/scripts/ai_core_manager.sh @@ -1,89 +1,182 @@ -#!/bin/bash +#!/usr/bin/env bash -# AI Core Manager for MobileOps Platform -# Manages AI inference engines, model loading, and resource allocation +# AI Core Manager for FileSystemds Platform +# Mobile-optimized AI inference, model management, and intelligent automation +# Supports on-device processing, edge AI, and cloud integration set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -LOG_FILE="/var/log/mobileops/ai_core_manager.log" -AI_CONFIG_DIR="/etc/mobileops/ai" -MODEL_CACHE_DIR="/var/cache/mobileops/models" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +export PROJECT_ROOT + +# Mobile-friendly directory structure +if [[ -n "${ANDROID_DATA:-}" ]] || [[ -n "${ANDROID_ROOT:-}" ]]; then + BASE_DIR="${ANDROID_DATA}/data/com.spiralgang.filesystemds" +else + BASE_DIR="${HOME}/.local/share/filesystemds" +fi + +LOG_DIR="$BASE_DIR/logs" +CONFIG_DIR="$BASE_DIR/config" +MODEL_CACHE_DIR="$BASE_DIR/cache/models" +AI_LIB_DIR="$BASE_DIR/lib/ai" + +LOG_FILE="$LOG_DIR/ai_core_manager.log" + +# Ensure directories exist +mkdir -p "$LOG_DIR" "$CONFIG_DIR" "$MODEL_CACHE_DIR" "$AI_LIB_DIR" log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE" + local level="${1:-INFO}" + shift + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" | tee -a "$LOG_FILE" } -start_ai_engine() { - local engine_type="$1" - log "INFO: Starting AI engine: $engine_type" +log_info() { log "INFO" "$@"; } +log_warn() { log "WARN" "$@"; } +log_error() { log "ERROR" "$@"; } +log_success() { log "SUCCESS" "$@"; } + +init_ai_core() { + log_info "Initializing AI Core Manager" - case "$engine_type" in - "neural-net") - log "INFO: Initializing neural network engine" - # Neural network engine startup - ;; - "llm") - log "INFO: Initializing large language model engine" - # LLM engine startup - ;; - "vision") - log "INFO: Initializing computer vision engine" - # Vision engine startup - ;; - *) - log "ERROR: Unknown AI engine type: $engine_type" - return 1 - ;; - esac -} + # Create AI configuration + cat > "$CONFIG_DIR/ai_core.conf" << EOF +# FileSystemds AI Core Configuration +ai_core_version=1.0.0 +initialization_time=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +# AI Engine Configuration +inference_backend=cpu +model_cache_size=1GB +max_concurrent_inferences=2 +memory_limit=512MB + +# Mobile optimizations +power_aware=true +battery_conservation=true +thermal_throttling=true +background_processing=false + +# Model management +auto_download_models=false +pointer_first_assets=true +require_explicit_fetch=true +EOF -load_model() { - local model_name="$1" - local model_path="$MODEL_CACHE_DIR/$model_name" + # Initialize model cache structure + mkdir -p "$MODEL_CACHE_DIR"/{small,medium,large} - log "INFO: Loading AI model: $model_name" + # Create model manifest + cat > "$MODEL_CACHE_DIR/manifest.json" << EOF +{ + "cache_version": "1.0.0", + "last_updated": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", + "models": {}, + "total_size": "0MB", + "cache_policy": "pointer-first" +} +EOF + + log_success "AI Core Manager initialized successfully" +} + +start_ai_core() { + log_info "Starting AI Core services" - if [[ ! -f "$model_path" ]]; then - log "ERROR: Model not found: $model_path" + if [[ ! -f "$CONFIG_DIR/ai_core.conf" ]]; then + log_error "AI Core not initialized. Run 'init' first." return 1 fi - log "INFO: Model $model_name loaded successfully" + # Start AI processing daemon (mock for now) + log_info "Starting AI inference engine" + + # Create PID file for tracking + echo $$ > "$BASE_DIR/ai_core.pid" + + log_success "AI Core services started" +} + +stop_ai_core() { + log_info "Stopping AI Core services" + + if [[ -f "$BASE_DIR/ai_core.pid" ]]; then + local pid + pid=$(cat "$BASE_DIR/ai_core.pid") + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" + rm -f "$BASE_DIR/ai_core.pid" + fi + fi + + log_success "AI Core services stopped" } -monitor_resources() { - log "INFO: Monitoring AI core resources" - # Resource monitoring logic - local gpu_usage=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null || echo "0") - local memory_usage=$(free | awk '/^Mem:/ {printf "%.1f", $3/$2 * 100.0}') +status_ai_core() { + if [[ ! -f "$CONFIG_DIR/ai_core.conf" ]]; then + echo "AI Core Status: NOT INITIALIZED" + return 1 + fi - log "INFO: GPU Usage: ${gpu_usage}%, Memory Usage: ${memory_usage}%" + if [[ -f "$BASE_DIR/ai_core.pid" ]]; then + echo "AI Core Status: RUNNING" + else + echo "AI Core Status: STOPPED" + fi +} + +show_help() { + cat << EOF +FileSystemds AI Core Manager - Mobile AI Platform + +USAGE: + $0 [options] + +COMMANDS: + init Initialize AI Core for first use + start Start AI processing services + stop Stop AI processing services + status Show AI Core status + help Show this help message + +EXAMPLES: + $0 init # Initialize AI Core + $0 start # Start AI services + $0 status # Check status + +EOF } main() { - mkdir -p "$(dirname "$LOG_FILE")" "$AI_CONFIG_DIR" "$MODEL_CACHE_DIR" - log "INFO: AI Core Manager started" + local command="${1:-help}" - case "${1:-status}" in - "start") - start_ai_engine "${2:-neural-net}" + case "$command" in + "init") + init_ai_core ;; - "load") - load_model "${2:-default.model}" + "start") + start_ai_core ;; - "monitor") - monitor_resources + "stop") + stop_ai_core ;; "status") - log "INFO: AI Core Manager is running" - monitor_resources + status_ai_core + ;; + "help"|"-h"|"--help") + show_help ;; *) - echo "Usage: $0 {start|load|monitor|status} [args]" + echo "Unknown command: $command" + echo "Run '$0 help' for usage information" exit 1 ;; esac } -main "$@" \ No newline at end of file +# Script entry point +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi \ No newline at end of file diff --git a/scripts/platform_launcher.sh b/scripts/platform_launcher.sh index e451c574b49..7c6ec2b4552 100755 --- a/scripts/platform_launcher.sh +++ b/scripts/platform_launcher.sh @@ -1,73 +1,257 @@ -#!/bin/bash +#!/usr/bin/env bash -# Platform Launcher Script for MobileOps Platform -# Main entry point for initializing and managing the MobileOps platform +# FileSystemds Platform Launcher - Mobile-First Architecture +# Advanced orchestration for Android/ARM64 mobile and edge deployment +# Supports containerized services, AI core management, and zero-trust security set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -LOG_FILE="/var/log/mobileops/platform_launcher.log" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +export PROJECT_ROOT + +# Mobile-friendly directory structure (user-space for Android) +if [[ -n "${ANDROID_DATA:-}" ]] || [[ -n "${ANDROID_ROOT:-}" ]]; then + # Android environment + BASE_DIR="${ANDROID_DATA}/data/com.spiralgang.filesystemds" + LOG_DIR="$BASE_DIR/logs" + CONFIG_DIR="$BASE_DIR/config" + CACHE_DIR="$BASE_DIR/cache" + LIB_DIR="$BASE_DIR/lib" +else + # Standard Linux environment + BASE_DIR="${HOME}/.local/share/filesystemds" + LOG_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/filesystemds/logs" + CONFIG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/filesystemds" + CACHE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/filesystemds" + LIB_DIR="$BASE_DIR/lib" +fi + +LOG_FILE="$LOG_DIR/platform_launcher.log" + +# Ensure directories exist +mkdir -p "$LOG_DIR" "$CONFIG_DIR" "$CACHE_DIR" "$LIB_DIR" log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE" + local level="${1:-INFO}" + shift + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" | tee -a "$LOG_FILE" +} + +log_info() { log "INFO" "$@"; } +log_warn() { log "WARN" "$@"; } +log_error() { log "ERROR" "$@"; } +log_success() { log "SUCCESS" "$@"; } + +detect_environment() { + local env_type="unknown" + local arch + arch="$(uname -m)" + + if [[ -n "${ANDROID_DATA:-}" ]]; then + env_type="android" + elif [[ -f "/etc/alpine-release" ]]; then + env_type="alpine" + elif [[ -f "/etc/debian_version" ]]; then + env_type="debian" + elif [[ -f "/etc/redhat-release" ]]; then + env_type="redhat" + elif [[ "$OSTYPE" == "darwin"* ]]; then + env_type="macos" + fi + + log_info "Environment detected: $env_type on $arch" + echo "$env_type:$arch" } initialize_platform() { - log "INFO: Initializing MobileOps Platform" + log_info "Initializing FileSystemds Mobile Platform" + + local env_info + env_info=$(detect_environment) + local env_type="${env_info%:*}" + local arch="${env_info#*:}" + + # Create platform configuration + cat > "$CONFIG_DIR/platform.conf" << EOF +# FileSystemds Platform Configuration +platform_version=1.0.0 +environment_type=$env_type +architecture=$arch +base_directory=$BASE_DIR +initialization_time=$(date -u +%Y-%m-%dT%H:%M:%SZ) +user_id=$(id -u) +group_id=$(id -g) + +# Feature flags for mobile deployment +mobile_optimized=true +container_runtime_enabled=true +ai_core_enabled=true +network_management_enabled=true +security_framework_enabled=true +EOF + + # Initialize all components with mobile-friendly approach + if [[ -x "$SCRIPT_DIR/ai_core_manager.sh" ]]; then + "$SCRIPT_DIR/ai_core_manager.sh" init + fi - # Create necessary directories - mkdir -p /var/log/mobileops - mkdir -p /etc/mobileops - mkdir -p /var/lib/mobileops - mkdir -p /var/run/mobileops + if [[ -x "$SCRIPT_DIR/network_configure.sh" ]]; then + "$SCRIPT_DIR/network_configure.sh" init + fi - # Initialize all components - "$SCRIPT_DIR/component_provisioner.sh" ai-core - "$SCRIPT_DIR/network_configure.sh" setup-container - "$SCRIPT_DIR/plugin_manager.sh" init - "$SCRIPT_DIR/asset_manager.sh" init + if [[ -x "$SCRIPT_DIR/asset_manager.sh" ]]; then + "$SCRIPT_DIR/asset_manager.sh" init + fi - log "INFO: MobileOps Platform initialized successfully" + log_success "FileSystemds Platform initialized successfully for $env_type on $arch" } start_platform() { - log "INFO: Starting MobileOps Platform services" + log_info "Starting FileSystemds Platform services" + + if [[ ! -f "$CONFIG_DIR/platform.conf" ]]; then + log_error "Platform not initialized. Run 'init' first." + exit 1 + fi + + # Start core services based on environment + local env_info + env_info=$(detect_environment) + local env_type="${env_info%:*}" + + case "$env_type" in + "android") + log_info "Starting Android-optimized services" + # Android-specific service startup + ;; + "alpine"|"debian"|"redhat") + log_info "Starting Linux services" + # Linux service startup + ;; + *) + log_warn "Unknown environment, starting basic services" + ;; + esac - # Start core services - "$SCRIPT_DIR/ai_core_manager.sh" start - "$SCRIPT_DIR/system_log_collector.sh" collect + # Start AI core if enabled + if [[ -x "$SCRIPT_DIR/ai_core_manager.sh" ]]; then + "$SCRIPT_DIR/ai_core_manager.sh" start || log_warn "AI Core Manager failed to start" + fi - log "INFO: MobileOps Platform services started" + log_success "FileSystemds Platform services started" } stop_platform() { - log "INFO: Stopping MobileOps Platform services" + log_info "Stopping FileSystemds Platform services" - # Stop all running components - pkill -f "mobileops" || true + # Graceful shutdown of components + if [[ -x "$SCRIPT_DIR/ai_core_manager.sh" ]]; then + "$SCRIPT_DIR/ai_core_manager.sh" stop || log_warn "AI Core Manager failed to stop gracefully" + fi - log "INFO: MobileOps Platform services stopped" + # Stop any remaining processes + pkill -f "filesystemds" || true + + log_success "FileSystemds Platform services stopped" } status_platform() { - log "INFO: Checking MobileOps Platform status" + log_info "Checking FileSystemds Platform status" + + if [[ ! -f "$CONFIG_DIR/platform.conf" ]]; then + echo "Platform Status: NOT INITIALIZED" + return 1 + fi + + local platform_version + platform_version=$(grep "platform_version=" "$CONFIG_DIR/platform.conf" | cut -d'=' -f2) + + echo "============================================" + echo "FileSystemds Platform Status" + echo "============================================" + echo "Version: $platform_version" + echo "Configuration: $CONFIG_DIR/platform.conf" + echo "Logs: $LOG_FILE" + echo "Base Directory: $BASE_DIR" + echo "Environment: $(detect_environment)" + echo "============================================" + + # Check service status + if pgrep -f "filesystemds" > /dev/null; then + echo "Platform Status: RUNNING" + else + echo "Platform Status: STOPPED" + fi +} + +health_check() { + log_info "Running FileSystemds Platform health check" + + local issues=0 - echo "=== MOBILEOPS PLATFORM STATUS ===" - echo "AI Core Status:" - "$SCRIPT_DIR/ai_core_manager.sh" status + # Check directory structure + for dir in "$LOG_DIR" "$CONFIG_DIR" "$CACHE_DIR" "$LIB_DIR"; do + if [[ ! -d "$dir" ]]; then + log_error "Missing directory: $dir" + ((issues++)) + fi + done - echo -e "\nNetwork Status:" - "$SCRIPT_DIR/network_configure.sh" monitor + # Check configuration + if [[ ! -f "$CONFIG_DIR/platform.conf" ]]; then + log_error "Missing platform configuration" + ((issues++)) + fi - echo -e "\nPlugin Status:" - "$SCRIPT_DIR/plugin_manager.sh" monitor + # Check script permissions + for script in "$SCRIPT_DIR"/*.sh; do + if [[ ! -x "$script" ]]; then + log_warn "Script not executable: $script" + fi + done + + if [[ $issues -eq 0 ]]; then + log_success "Health check passed - no issues found" + return 0 + else + log_error "Health check failed - $issues issues found" + return 1 + fi +} + +show_help() { + cat << EOF +FileSystemds Platform Launcher - Mobile-First Architecture + +USAGE: + $0 [options] + +COMMANDS: + init Initialize the platform for first use + start Start all platform services + stop Stop all platform services + restart Restart all platform services + status Show platform status and configuration + health Run platform health check + logs Show recent platform logs + cleanup Clean up temporary files and caches + help Show this help message + +EXAMPLES: + $0 init # Initialize for first use + $0 start # Start all services + $0 status # Check status + $0 health # Run health check + +For more information, visit: https://github.com/spiralgang/FileSystemds +EOF } main() { - mkdir -p "$(dirname "$LOG_FILE")" - log "INFO: Platform Launcher started" + local command="${1:-help}" - case "${1:-status}" in + case "$command" in "init") initialize_platform ;; @@ -85,11 +269,33 @@ main() { "status") status_platform ;; + "health") + health_check + ;; + "logs") + if [[ -f "$LOG_FILE" ]]; then + tail -n 50 "$LOG_FILE" + else + echo "No logs found at $LOG_FILE" + fi + ;; + "cleanup") + log_info "Cleaning up temporary files" + rm -rf "${CACHE_DIR:?}"/* || true + log_success "Cleanup completed" + ;; + "help"|"-h"|"--help") + show_help + ;; *) - echo "Usage: $0 {init|start|stop|restart|status}" + echo "Unknown command: $command" + echo "Run '$0 help' for usage information" exit 1 ;; esac } -main "$@" +# Script entry point +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi