From fddc15195ed087d32395c726490a6205384dce23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Sep 2025 18:11:21 -0400 Subject: [PATCH 01/19] chore(deps): bump next from 15.2.4 to 15.5.3 in /ui (#392) Bumps [next](https://github.com/vercel/next.js) from 15.2.4 to 15.5.3. - [Release notes](https://github.com/vercel/next.js/releases) - [Changelog](https://github.com/vercel/next.js/blob/canary/release.js) - [Commits](https://github.com/vercel/next.js/compare/v15.2.4...v15.5.3) --- updated-dependencies: - dependency-name: next dependency-version: 15.5.3 dependency-type: direct:production update-type: version-update:semver-minor ... --- ui/package-lock.json | 369 +++++++++++++++++++++++-------------------- ui/package.json | 2 +- 2 files changed, 202 insertions(+), 169 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index d8f807e8..1261f842 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -20,7 +20,7 @@ "clsx": "^2.1.1", "idb-keyval": "^6.2.1", "lucide-react": "^0.454.0", - "next": "15.2.4", + "next": "15.5.3", "next-themes": "^0.4.4", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -60,9 +60,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz", - "integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.5.0.tgz", + "integrity": "sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==", "license": "MIT", "optional": true, "dependencies": { @@ -218,9 +218,9 @@ "license": "BSD-3-Clause" }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", - "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.3.tgz", + "integrity": "sha512-ryFMfvxxpQRsgZJqBd4wsttYQbCxsJksrv9Lw/v798JcQ8+w84mBWuXwl+TT0WJ/WrYOLaYpwQXi3sA9nTIaIg==", "cpu": [ "arm64" ], @@ -236,13 +236,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.4" + "@img/sharp-libvips-darwin-arm64": "1.2.0" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", - "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.3.tgz", + "integrity": "sha512-yHpJYynROAj12TA6qil58hmPmAwxKKC7reUqtGLzsOHfP7/rniNGTL8tjWX6L3CTV4+5P4ypcS7Pp+7OB+8ihA==", "cpu": [ "x64" ], @@ -258,13 +258,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.0.4" + "@img/sharp-libvips-darwin-x64": "1.2.0" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", - "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.0.tgz", + "integrity": "sha512-sBZmpwmxqwlqG9ueWFXtockhsxefaV6O84BMOrhtg/YqbTaRdqDE7hxraVE3y6gVM4eExmfzW4a8el9ArLeEiQ==", "cpu": [ "arm64" ], @@ -278,9 +278,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", - "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.0.tgz", + "integrity": "sha512-M64XVuL94OgiNHa5/m2YvEQI5q2cl9d/wk0qFTDVXcYzi43lxuiFTftMR1tOnFQovVXNZJ5TURSDK2pNe9Yzqg==", "cpu": [ "x64" ], @@ -294,9 +294,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", - "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.0.tgz", + "integrity": "sha512-mWd2uWvDtL/nvIzThLq3fr2nnGfyr/XMXlq8ZJ9WMR6PXijHlC3ksp0IpuhK6bougvQrchUAfzRLnbsen0Cqvw==", "cpu": [ "arm" ], @@ -310,9 +310,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", - "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.0.tgz", + "integrity": "sha512-RXwd0CgG+uPRX5YYrkzKyalt2OJYRiJQ8ED/fi1tq9WQW2jsQIn0tqrlR5l5dr/rjqq6AHAxURhj2DVjyQWSOA==", "cpu": [ "arm64" ], @@ -325,10 +325,26 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.0.tgz", + "integrity": "sha512-Xod/7KaDDHkYu2phxxfeEPXfVXFKx70EAFZ0qyUdOjCcxbjqyJOEUpDe6RIyaunGxT34Anf9ue/wuWOqBW2WcQ==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", - "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.0.tgz", + "integrity": "sha512-eMKfzDxLGT8mnmPJTNMcjfO33fLiTDsrMlUVcp6b96ETbnJmd4uvZxVJSKPQfS+odwfVaGifhsB07J1LynFehw==", "cpu": [ "s390x" ], @@ -342,9 +358,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", - "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.0.tgz", + "integrity": "sha512-ZW3FPWIc7K1sH9E3nxIGB3y3dZkpJlMnkk7z5tu1nSkBoCgw2nSRTFHI5pB/3CQaJM0pdzMF3paf9ckKMSE9Tg==", "cpu": [ "x64" ], @@ -358,9 +374,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", - "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.0.tgz", + "integrity": "sha512-UG+LqQJbf5VJ8NWJ5Z3tdIe/HXjuIdo4JeVNADXBFuG7z9zjoegpzzGIyV5zQKi4zaJjnAd2+g2nna8TZvuW9Q==", "cpu": [ "arm64" ], @@ -374,9 +390,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", - "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.0.tgz", + "integrity": "sha512-SRYOLR7CXPgNze8akZwjoGBoN1ThNZoqpOgfnOxmWsklTGVfJiGJoC/Lod7aNMGA1jSsKWM1+HRX43OP6p9+6Q==", "cpu": [ "x64" ], @@ -390,9 +406,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", - "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.3.tgz", + "integrity": "sha512-oBK9l+h6KBN0i3dC8rYntLiVfW8D8wH+NPNT3O/WBHeW0OQWCjfWksLUaPidsrDKpJgXp3G3/hkmhptAW0I3+A==", "cpu": [ "arm" ], @@ -408,13 +424,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.0.5" + "@img/sharp-libvips-linux-arm": "1.2.0" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", - "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.3.tgz", + "integrity": "sha512-QdrKe3EvQrqwkDrtuTIjI0bu6YEJHTgEeqdzI3uWJOH6G1O8Nl1iEeVYRGdj1h5I21CqxSvQp1Yv7xeU3ZewbA==", "cpu": [ "arm64" ], @@ -430,13 +446,35 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.0.4" + "@img/sharp-libvips-linux-arm64": "1.2.0" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.3.tgz", + "integrity": "sha512-GLtbLQMCNC5nxuImPR2+RgrviwKwVql28FWZIW1zWruy6zLgA5/x2ZXk3mxj58X/tszVF69KK0Is83V8YgWhLA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.0" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", - "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.3.tgz", + "integrity": "sha512-3gahT+A6c4cdc2edhsLHmIOXMb17ltffJlxR0aC2VPZfwKoTGZec6u5GrFgdR7ciJSsHT27BD3TIuGcuRT0KmQ==", "cpu": [ "s390x" ], @@ -452,13 +490,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.0.4" + "@img/sharp-libvips-linux-s390x": "1.2.0" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", - "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.3.tgz", + "integrity": "sha512-8kYso8d806ypnSq3/Ly0QEw90V5ZoHh10yH0HnrzOCr6DKAPI6QVHvwleqMkVQ0m+fc7EH8ah0BB0QPuWY6zJQ==", "cpu": [ "x64" ], @@ -474,13 +512,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.0.4" + "@img/sharp-libvips-linux-x64": "1.2.0" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", - "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.3.tgz", + "integrity": "sha512-vAjbHDlr4izEiXM1OTggpCcPg9tn4YriK5vAjowJsHwdBIdx0fYRsURkxLG2RLm9gyBq66gwtWI8Gx0/ov+JKQ==", "cpu": [ "arm64" ], @@ -496,13 +534,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + "@img/sharp-libvips-linuxmusl-arm64": "1.2.0" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", - "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.3.tgz", + "integrity": "sha512-gCWUn9547K5bwvOn9l5XGAEjVTTRji4aPTqLzGXHvIr6bIDZKNTA34seMPgM0WmSf+RYBH411VavCejp3PkOeQ==", "cpu": [ "x64" ], @@ -518,20 +556,20 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + "@img/sharp-libvips-linuxmusl-x64": "1.2.0" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", - "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.3.tgz", + "integrity": "sha512-+CyRcpagHMGteySaWos8IbnXcHgfDn7pO2fiC2slJxvNq9gDipYBN42/RagzctVRKgxATmfqOSulgZv5e1RdMg==", "cpu": [ "wasm32" ], "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { - "@emnapi/runtime": "^1.2.0" + "@emnapi/runtime": "^1.4.4" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0" @@ -540,10 +578,29 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.3.tgz", + "integrity": "sha512-MjnHPnbqMXNC2UgeLJtX4XqoVHHlZNd+nPt1kRPmj63wURegwBhZlApELdtxM2OIZDRv/DFtLcNhVbd1z8GYXQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", - "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.3.tgz", + "integrity": "sha512-xuCdhH44WxuXgOM714hn4amodJMZl3OEvf0GVTm0BEyMeA2to+8HEdRPShH0SLYptJY1uBw+SCFP9WVQi1Q/cw==", "cpu": [ "ia32" ], @@ -560,9 +617,9 @@ } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", - "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.3.tgz", + "integrity": "sha512-OWwz05d++TxzLEv4VnsTz5CmZ6mI6S05sfQGEMrNrQcOEERbX46332IvE7pO/EUiw7jUrrS40z/M7kPyjfl04g==", "cpu": [ "x64" ], @@ -717,9 +774,9 @@ } }, "node_modules/@next/env": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/@next/env/-/env-15.2.4.tgz", - "integrity": "sha512-+SFtMgoiYP3WoSswuNmxJOCwi06TdWE733D+WPjpXIe4LXGULwEaofiiAy6kbS0+XjM5xF5n3lKuBwN2SnqD9g==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.3.tgz", + "integrity": "sha512-RSEDTRqyihYXygx/OJXwvVupfr9m04+0vH8vyy0HfZ7keRto6VX9BbEk0J2PUk0VGy6YhklJUSrgForov5F9pw==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { @@ -733,9 +790,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.2.4.tgz", - "integrity": "sha512-1AnMfs655ipJEDC/FHkSr0r3lXBgpqKo4K1kiwfUf3iE68rDFXZ1TtHdMvf7D0hMItgDZ7Vuq3JgNMbt/+3bYw==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.3.tgz", + "integrity": "sha512-nzbHQo69+au9wJkGKTU9lP7PXv0d1J5ljFpvb+LnEomLtSbJkbZyEs6sbF3plQmiOB2l9OBtN2tNSvCH1nQ9Jg==", "cpu": [ "arm64" ], @@ -749,9 +806,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.2.4.tgz", - "integrity": "sha512-3qK2zb5EwCwxnO2HeO+TRqCubeI/NgCe+kL5dTJlPldV/uwCnUgC7VbEzgmxbfrkbjehL4H9BPztWOEtsoMwew==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.3.tgz", + "integrity": "sha512-w83w4SkOOhekJOcA5HBvHyGzgV1W/XvOfpkrxIse4uPWhYTTRwtGEM4v/jiXwNSJvfRvah0H8/uTLBKRXlef8g==", "cpu": [ "x64" ], @@ -765,9 +822,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.2.4.tgz", - "integrity": "sha512-HFN6GKUcrTWvem8AZN7tT95zPb0GUGv9v0d0iyuTb303vbXkkbHDp/DxufB04jNVD+IN9yHy7y/6Mqq0h0YVaQ==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.3.tgz", + "integrity": "sha512-+m7pfIs0/yvgVu26ieaKrifV8C8yiLe7jVp9SpcIzg7XmyyNE7toC1fy5IOQozmr6kWl/JONC51osih2RyoXRw==", "cpu": [ "arm64" ], @@ -781,9 +838,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.2.4.tgz", - "integrity": "sha512-Oioa0SORWLwi35/kVB8aCk5Uq+5/ZIumMK1kJV+jSdazFm2NzPDztsefzdmzzpx5oGCJ6FkUC7vkaUseNTStNA==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.3.tgz", + "integrity": "sha512-u3PEIzuguSenoZviZJahNLgCexGFhso5mxWCrrIMdvpZn6lkME5vc/ADZG8UUk5K1uWRy4hqSFECrON6UKQBbQ==", "cpu": [ "arm64" ], @@ -797,9 +854,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.2.4.tgz", - "integrity": "sha512-yb5WTRaHdkgOqFOZiu6rHV1fAEK0flVpaIN2HB6kxHVSy/dIajWbThS7qON3W9/SNOH2JWkVCyulgGYekMePuw==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.3.tgz", + "integrity": "sha512-lDtOOScYDZxI2BENN9m0pfVPJDSuUkAD1YXSvlJF0DKwZt0WlA7T7o3wrcEr4Q+iHYGzEaVuZcsIbCps4K27sA==", "cpu": [ "x64" ], @@ -813,9 +870,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.2.4.tgz", - "integrity": "sha512-Dcdv/ix6srhkM25fgXiyOieFUkz+fOYkHlydWCtB0xMST6X9XYI3yPDKBZt1xuhOytONsIFJFB08xXYsxUwJLw==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.3.tgz", + "integrity": "sha512-9vWVUnsx9PrY2NwdVRJ4dUURAQ8Su0sLRPqcCCxtX5zIQUBES12eRVHq6b70bbfaVaxIDGJN2afHui0eDm+cLg==", "cpu": [ "x64" ], @@ -829,9 +886,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.2.4.tgz", - "integrity": "sha512-dW0i7eukvDxtIhCYkMrZNQfNicPDExt2jPb9AZPpL7cfyUo7QSNl1DjsHjmmKp6qNAqUESyT8YFl/Aw91cNJJg==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.3.tgz", + "integrity": "sha512-1CU20FZzY9LFQigRi6jM45oJMU3KziA5/sSG+dXeVaTm661snQP6xu3ykGxxwU5sLG3sh14teO/IOEPVsQMRfA==", "cpu": [ "arm64" ], @@ -845,9 +902,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.2.4.tgz", - "integrity": "sha512-SbnWkJmkS7Xl3kre8SdMF6F/XDh1DTFEhp0jRTj/uB8iPKoU2bb2NDfcu+iifv1+mxQEd1g2vvSxcZbXSKyWiQ==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.3.tgz", + "integrity": "sha512-JMoLAq3n3y5tKXPQwCK5c+6tmwkuFDa2XAxz8Wm4+IVthdBZdZGh+lmiLUHg9f9IDwIQpUjp+ysd6OkYTyZRZw==", "cpu": [ "x64" ], @@ -2117,12 +2174,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@swc/counter": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", - "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==", - "license": "Apache-2.0" - }, "node_modules/@swc/helpers": { "version": "0.5.15", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", @@ -2860,17 +2911,6 @@ "ieee754": "^1.2.1" } }, - "node_modules/busboy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", - "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", - "dependencies": { - "streamsearch": "^1.1.0" - }, - "engines": { - "node": ">=10.16.0" - } - }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -3315,9 +3355,9 @@ } }, "node_modules/detect-libc": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", - "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", + "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", "license": "Apache-2.0", "optional": true, "engines": { @@ -5695,15 +5735,13 @@ "license": "MIT" }, "node_modules/next": { - "version": "15.2.4", - "resolved": "https://registry.npmjs.org/next/-/next-15.2.4.tgz", - "integrity": "sha512-VwL+LAaPSxEkd3lU2xWbgEOtrM8oedmyhBqaVNmgKB+GvZlCy9rgaEc+y2on0wv+l0oSFqLtYD6dcC1eAedUaQ==", + "version": "15.5.3", + "resolved": "https://registry.npmjs.org/next/-/next-15.5.3.tgz", + "integrity": "sha512-r/liNAx16SQj4D+XH/oI1dlpv9tdKJ6cONYPwwcCC46f2NjpaRWY+EKCzULfgQYV6YKXjHBchff2IZBSlZmJNw==", "license": "MIT", "dependencies": { - "@next/env": "15.2.4", - "@swc/counter": "0.1.3", + "@next/env": "15.5.3", "@swc/helpers": "0.5.15", - "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" @@ -5715,19 +5753,19 @@ "node": "^18.18.0 || ^19.8.0 || >= 20.0.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "15.2.4", - "@next/swc-darwin-x64": "15.2.4", - "@next/swc-linux-arm64-gnu": "15.2.4", - "@next/swc-linux-arm64-musl": "15.2.4", - "@next/swc-linux-x64-gnu": "15.2.4", - "@next/swc-linux-x64-musl": "15.2.4", - "@next/swc-win32-arm64-msvc": "15.2.4", - "@next/swc-win32-x64-msvc": "15.2.4", - "sharp": "^0.33.5" + "@next/swc-darwin-arm64": "15.5.3", + "@next/swc-darwin-x64": "15.5.3", + "@next/swc-linux-arm64-gnu": "15.5.3", + "@next/swc-linux-arm64-musl": "15.5.3", + "@next/swc-linux-x64-gnu": "15.5.3", + "@next/swc-linux-x64-musl": "15.5.3", + "@next/swc-win32-arm64-msvc": "15.5.3", + "@next/swc-win32-x64-msvc": "15.5.3", + "sharp": "^0.34.3" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", - "@playwright/test": "^1.41.2", + "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", @@ -6758,9 +6796,9 @@ "license": "MIT" }, "node_modules/semver": { - "version": "7.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", - "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", "devOptional": true, "license": "ISC", "bin": { @@ -6820,16 +6858,16 @@ } }, "node_modules/sharp": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", - "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "version": "0.34.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.3.tgz", + "integrity": "sha512-eX2IQ6nFohW4DbvHIOLRB3MHFpYqaqvXd3Tp5e/T/dSH83fxaNJQRvDMhASmkNTsNTVF2/OOopzRCt7xokgPfg==", "hasInstallScript": true, "license": "Apache-2.0", "optional": true, "dependencies": { "color": "^4.2.3", - "detect-libc": "^2.0.3", - "semver": "^7.6.3" + "detect-libc": "^2.0.4", + "semver": "^7.7.2" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0" @@ -6838,25 +6876,28 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.5", - "@img/sharp-darwin-x64": "0.33.5", - "@img/sharp-libvips-darwin-arm64": "1.0.4", - "@img/sharp-libvips-darwin-x64": "1.0.4", - "@img/sharp-libvips-linux-arm": "1.0.5", - "@img/sharp-libvips-linux-arm64": "1.0.4", - "@img/sharp-libvips-linux-s390x": "1.0.4", - "@img/sharp-libvips-linux-x64": "1.0.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", - "@img/sharp-libvips-linuxmusl-x64": "1.0.4", - "@img/sharp-linux-arm": "0.33.5", - "@img/sharp-linux-arm64": "0.33.5", - "@img/sharp-linux-s390x": "0.33.5", - "@img/sharp-linux-x64": "0.33.5", - "@img/sharp-linuxmusl-arm64": "0.33.5", - "@img/sharp-linuxmusl-x64": "0.33.5", - "@img/sharp-wasm32": "0.33.5", - "@img/sharp-win32-ia32": "0.33.5", - "@img/sharp-win32-x64": "0.33.5" + "@img/sharp-darwin-arm64": "0.34.3", + "@img/sharp-darwin-x64": "0.34.3", + "@img/sharp-libvips-darwin-arm64": "1.2.0", + "@img/sharp-libvips-darwin-x64": "1.2.0", + "@img/sharp-libvips-linux-arm": "1.2.0", + "@img/sharp-libvips-linux-arm64": "1.2.0", + "@img/sharp-libvips-linux-ppc64": "1.2.0", + "@img/sharp-libvips-linux-s390x": "1.2.0", + "@img/sharp-libvips-linux-x64": "1.2.0", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.0", + "@img/sharp-libvips-linuxmusl-x64": "1.2.0", + "@img/sharp-linux-arm": "0.34.3", + "@img/sharp-linux-arm64": "0.34.3", + "@img/sharp-linux-ppc64": "0.34.3", + "@img/sharp-linux-s390x": "0.34.3", + "@img/sharp-linux-x64": "0.34.3", + "@img/sharp-linuxmusl-arm64": "0.34.3", + "@img/sharp-linuxmusl-x64": "0.34.3", + "@img/sharp-wasm32": "0.34.3", + "@img/sharp-win32-arm64": "0.34.3", + "@img/sharp-win32-ia32": "0.34.3", + "@img/sharp-win32-x64": "0.34.3" } }, "node_modules/shebang-command": { @@ -7034,14 +7075,6 @@ "dev": true, "license": "MIT" }, - "node_modules/streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/string-argv": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz", diff --git a/ui/package.json b/ui/package.json index ef3ae437..666c6a5f 100644 --- a/ui/package.json +++ b/ui/package.json @@ -24,7 +24,7 @@ "clsx": "^2.1.1", "idb-keyval": "^6.2.1", "lucide-react": "^0.454.0", - "next": "15.2.4", + "next": "15.5.3", "next-themes": "^0.4.4", "react": "^19.0.0", "react-dom": "^19.0.0", From bc719dc5326e812d75b0b7abd4a8db800a635afa Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Wed, 17 Sep 2025 17:11:16 -0400 Subject: [PATCH 02/19] workflows: fix sample rate in audio transcription (#408) --- workflows/comfystream/audio-transcription-api.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workflows/comfystream/audio-transcription-api.json b/workflows/comfystream/audio-transcription-api.json index 3865a6fb..f80b94d7 100644 --- a/workflows/comfystream/audio-transcription-api.json +++ b/workflows/comfystream/audio-transcription-api.json @@ -1,7 +1,10 @@ { "10": { "inputs": { - "sample_rate": 16000, + "sample_rate": [ + "11", + 1 + ], "transcription_interval": 2, "accumulation_duration": 3, "whisper_model": "base", From ad301a4c2efca54c17fb583e063c54f321537a9e Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Fri, 19 Sep 2025 19:09:04 -0400 Subject: [PATCH 03/19] feat(byoc): add trickle to comfystream (#357) * Add BYOC Server integration to comfystream * client: add cleanup to set_prompts. clear queues on prompt change * frame_processor: add optional preprocess to put_audio_frame to fix pipeline compatibility * Update output type to AUDIO for LoadAudioTensor/PitchShifter/SaveAudioTensor to match LoadAudioTensor * normalize audio in pitch shifter node --- .vscode/launch.json | 36 ++- nodes/audio_utils/load_audio_tensor.py | 26 +- nodes/audio_utils/pitch_shift.py | 49 ++- nodes/audio_utils/save_audio_tensor.py | 26 +- nodes/tensor_utils/save_text_tensor.py | 2 +- pyproject.toml | 1 + requirements.txt | 1 + server/app.py | 14 +- server/byoc.py | 211 +++++++++++++ server/frame_processor.py | 280 ++++++++++++++++++ src/comfystream/client.py | 24 +- src/comfystream/pipeline.py | 42 ++- src/comfystream/utils.py | 86 +++++- .../audio-tensor-utils-example-api.json | 66 ++--- .../comfystream/audio-transcription-api.json | 6 +- 15 files changed, 788 insertions(+), 82 deletions(-) create mode 100644 server/byoc.py create mode 100644 server/frame_processor.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 540197b1..4d442c58 100755 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -31,10 +31,40 @@ "--media-ports=5678", "--host=0.0.0.0", "--port=8889", - "--log-level=DEBUG", + "--log-level=INFO", + "--comfyui-inference-log-level=DEBUG", ], - "python": "/workspace/miniconda3/envs/comfystream/bin/python", - "justMyCode": true + "justMyCode": true, + "python": "${command:python.interpreterPath}" + }, + { + "name": "Run ComfyStream BYOC", + "type": "debugpy", + "request": "launch", + "cwd": "/workspace/ComfyUI", + "program": "/workspace/comfystream/server/byoc.py", + "console": "integratedTerminal", + "args": [ + "--workspace=/workspace/ComfyUI", + "--host=0.0.0.0", + "--port=8000", + "--log-level=INFO", + "--comfyui-inference-log-level=DEBUG", + "--width=512", + "--height=512" + ], + "env": { + "ORCH_URL": "https://172.17.0.1:9995", + "ORCH_SECRET": "orch-secret", + "CAPABILITY_NAME": "comfystream-byoc-processor", + "CAPABILITY_DESCRIPTION": "ComfyUI streaming processor for BYOC mode", + "CAPABILITY_URL": "http://172.17.0.1:8000", + "CAPABILITY_PRICE_PER_UNIT": "0", + "CAPABILITY_PRICE_SCALING": "1", + "CAPABILITY_CAPACITY": "1" + }, + "justMyCode": true, + "python": "${command:python.interpreterPath}" }, { "name": "Run ComfyStream UI (Node.js)", diff --git a/nodes/audio_utils/load_audio_tensor.py b/nodes/audio_utils/load_audio_tensor.py index 52fa6fa3..ece7fca3 100644 --- a/nodes/audio_utils/load_audio_tensor.py +++ b/nodes/audio_utils/load_audio_tensor.py @@ -1,11 +1,12 @@ import numpy as np +import torch from comfystream import tensor_cache class LoadAudioTensor: CATEGORY = "audio_utils" - RETURN_TYPES = ("WAVEFORM", "INT") - RETURN_NAMES = ("audio", "sample_rate") + RETURN_TYPES = ("AUDIO",) + RETURN_NAMES = ("audio",) FUNCTION = "execute" def __init__(self): @@ -22,7 +23,7 @@ def INPUT_TYPES(s): } @classmethod - def IS_CHANGED(): + def IS_CHANGED(**kwargs): return float("nan") def execute(self, buffer_size): @@ -50,4 +51,21 @@ def execute(self, buffer_size): buffered_audio = self.leftover[:self.buffer_samples] self.leftover = self.leftover[self.buffer_samples:] - return buffered_audio, self.sample_rate + # Convert numpy array to torch tensor and normalize int16 to float32 + waveform_tensor = torch.from_numpy(buffered_audio.astype(np.float32) / 32768.0) + + # Ensure proper tensor shape: (batch, channels, samples) + if waveform_tensor.dim() == 1: + # Mono: (samples,) -> (1, 1, samples) + waveform_tensor = waveform_tensor.unsqueeze(0).unsqueeze(0) + elif waveform_tensor.dim() == 2: + # Assume (channels, samples) and add batch dimension + waveform_tensor = waveform_tensor.unsqueeze(0) + + # Return AUDIO dictionary format + audio_dict = { + "waveform": waveform_tensor, + "sample_rate": self.sample_rate + } + + return (audio_dict,) diff --git a/nodes/audio_utils/pitch_shift.py b/nodes/audio_utils/pitch_shift.py index ed2b2b38..2fba9ee5 100644 --- a/nodes/audio_utils/pitch_shift.py +++ b/nodes/audio_utils/pitch_shift.py @@ -1,17 +1,17 @@ import numpy as np import librosa +import torch class PitchShifter: CATEGORY = "audio_utils" - RETURN_TYPES = ("WAVEFORM", "INT") + RETURN_TYPES = ("AUDIO",) FUNCTION = "execute" @classmethod def INPUT_TYPES(cls): return { "required": { - "audio": ("WAVEFORM",), - "sample_rate": ("INT",), + "audio": ("AUDIO",), "pitch_shift": ("FLOAT", { "default": 4.0, "min": 0.0, @@ -25,8 +25,41 @@ def INPUT_TYPES(cls): def IS_CHANGED(cls): return float("nan") - def execute(self, audio, sample_rate, pitch_shift): - audio_float = audio.astype(np.float32) / 32768.0 - shifted_audio = librosa.effects.pitch_shift(y=audio_float, sr=sample_rate, n_steps=pitch_shift) - shifted_int16 = np.clip(shifted_audio * 32768.0, -32768, 32767).astype(np.int16) - return shifted_int16, sample_rate + def execute(self, audio, pitch_shift): + # Extract waveform and sample rate from AUDIO format + waveform = audio["waveform"] + sample_rate = audio["sample_rate"] + + # Convert tensor to numpy and ensure proper format for librosa + if isinstance(waveform, torch.Tensor): + audio_numpy = waveform.squeeze().cpu().numpy() + else: + audio_numpy = waveform.squeeze() + + # Ensure float32 format and proper normalization for librosa processing + if audio_numpy.dtype != np.float32: + audio_numpy = audio_numpy.astype(np.float32) + + # Check if data needs normalization (librosa expects [-1, 1] range) + max_abs_val = np.abs(audio_numpy).max() + if max_abs_val > 1.0: + # Data appears to be in int16 range, normalize it + audio_numpy = audio_numpy / 32768.0 + + # Apply pitch shift + shifted_audio = librosa.effects.pitch_shift(y=audio_numpy, sr=sample_rate, n_steps=pitch_shift) + + # Convert back to tensor and restore original shape + shifted_tensor = torch.from_numpy(shifted_audio).float() + if waveform.dim() == 3: # (batch, channels, samples) + shifted_tensor = shifted_tensor.unsqueeze(0).unsqueeze(0) + elif waveform.dim() == 2: # (channels, samples) + shifted_tensor = shifted_tensor.unsqueeze(0) + + # Return AUDIO format + result_audio = { + "waveform": shifted_tensor, + "sample_rate": sample_rate + } + + return (result_audio,) diff --git a/nodes/audio_utils/save_audio_tensor.py b/nodes/audio_utils/save_audio_tensor.py index 6f86b57c..6b7b0281 100644 --- a/nodes/audio_utils/save_audio_tensor.py +++ b/nodes/audio_utils/save_audio_tensor.py @@ -1,3 +1,4 @@ +import numpy as np from comfystream import tensor_cache class SaveAudioTensor: @@ -11,7 +12,7 @@ class SaveAudioTensor: def INPUT_TYPES(s): return { "required": { - "audio": ("WAVEFORM",) + "audio": ("AUDIO",) } } @@ -20,5 +21,26 @@ def IS_CHANGED(s): return float("nan") def execute(self, audio): - tensor_cache.audio_outputs.put_nowait(audio) + # Extract waveform tensor from AUDIO format + waveform = audio["waveform"] + + # Convert to numpy and flatten for pipeline compatibility + if hasattr(waveform, 'cpu'): + # PyTorch tensor + waveform_numpy = waveform.squeeze().cpu().numpy() + else: + # Already numpy + waveform_numpy = waveform.squeeze() + + # Ensure 1D array for pipeline buffer concatenation + if waveform_numpy.ndim > 1: + waveform_numpy = waveform_numpy.flatten() + + # Convert to int16 if needed (pipeline expects int16) + if waveform_numpy.dtype == np.float32: + waveform_numpy = (waveform_numpy * 32767).astype(np.int16) + elif waveform_numpy.dtype != np.int16: + waveform_numpy = waveform_numpy.astype(np.int16) + + tensor_cache.audio_outputs.put_nowait(waveform_numpy) return (audio,) diff --git a/nodes/tensor_utils/save_text_tensor.py b/nodes/tensor_utils/save_text_tensor.py index 525f2a1b..098887e0 100644 --- a/nodes/tensor_utils/save_text_tensor.py +++ b/nodes/tensor_utils/save_text_tensor.py @@ -18,7 +18,7 @@ def INPUT_TYPES(s): } @classmethod - def IS_CHANGED(s): + def IS_CHANGED(s, **kwargs): return float("nan") def execute(self, data, remove_linebreaks=True): diff --git a/pyproject.toml b/pyproject.toml index 8cc48645..50e59935 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ version = "0.1.5" license = { file = "LICENSE" } dependencies = [ "asyncio", + "pytrickle @ git+https://github.com/livepeer/pytrickle.git@de37bea74679fa5db46b656a83c9b7240fc597b6", "comfyui @ git+https://github.com/hiddenswitch/ComfyUI.git@58622c7e91cb5cc2bca985d713db55e5681ff316", "aiortc", "aiohttp", diff --git a/requirements.txt b/requirements.txt index 7ff3310b..790900bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ asyncio +pytrickle @ git+https://github.com/livepeer/pytrickle.git@de37bea74679fa5db46b656a83c9b7240fc597b6 comfyui @ git+https://github.com/hiddenswitch/ComfyUI.git@58622c7e91cb5cc2bca985d713db55e5681ff316 aiortc aiohttp diff --git a/server/app.py b/server/app.py index ecf5751f..a3a42fc4 100644 --- a/server/app.py +++ b/server/app.py @@ -40,6 +40,7 @@ MAX_BITRATE = 2000000 MIN_BITRATE = 2000000 +TEXT_POLL_INTERVAL = 0.25 # Interval in seconds to poll for text outputs class VideoStreamTrack(MediaStreamTrack): @@ -390,11 +391,11 @@ async def forward_text(): try: while channel.readyState == "open": try: - # Use timeout to prevent indefinite blocking - text = await asyncio.wait_for( - pipeline.get_text_output(), - timeout=1.0 # Check every second if channel is still open - ) + # Non-blocking poll; sleep if no text to avoid tight loop + text = await pipeline.get_text_output() + if text is None or text.strip() == "": + await asyncio.sleep(TEXT_POLL_INTERVAL) + continue if channel.readyState == "open": # Send as JSON string for extensibility try: @@ -402,9 +403,6 @@ async def forward_text(): except Exception as e: logger.debug(f"[TextChannel] Send failed, stopping forwarder: {e}") break - except asyncio.TimeoutError: - # No text available, continue checking - continue except asyncio.CancelledError: logger.debug("[TextChannel] Forward text task cancelled") break diff --git a/server/byoc.py b/server/byoc.py new file mode 100644 index 00000000..0735674b --- /dev/null +++ b/server/byoc.py @@ -0,0 +1,211 @@ +import argparse +import asyncio +import logging +import os +import sys + +import torch +# Initialize CUDA before any other imports to prevent core dump. +if torch.cuda.is_available(): + torch.cuda.init() + +from aiohttp import web +from pytrickle.stream_processor import StreamProcessor +from pytrickle.utils.register import RegisterCapability +from pytrickle.frame_skipper import FrameSkipConfig +from frame_processor import ComfyStreamFrameProcessor + +logger = logging.getLogger(__name__) + + +async def register_orchestrator(orch_url=None, orch_secret=None, capability_name=None, host="127.0.0.1", port=8889): + """Register capability with orchestrator if configured.""" + try: + orch_url = orch_url or os.getenv("ORCH_URL") + orch_secret = orch_secret or os.getenv("ORCH_SECRET") + + if orch_url and orch_secret: + os.environ.update({ + "CAPABILITY_NAME": capability_name or os.getenv("CAPABILITY_NAME") or "comfystream-processor", + "CAPABILITY_DESCRIPTION": "ComfyUI streaming processor", + "CAPABILITY_URL": f"http://{host}:{port}", + "CAPABILITY_CAPACITY": "1", + "ORCH_URL": orch_url, + "ORCH_SECRET": orch_secret + }) + + # Pass through explicit capability_name to ensure CLI/env override takes effect + result = await RegisterCapability.register( + logger=logger, + capability_name=capability_name + ) + if result: + logger.info(f"Registered capability: {result.geturl()}") + except Exception as e: + logger.error(f"Orchestrator registration failed: {e}") + + +def main(): + parser = argparse.ArgumentParser( + description="Run comfystream server in BYOC (Bring Your Own Compute) mode using pytrickle." + ) + parser.add_argument("--port", default=8889, help="Set the server port") + parser.add_argument("--host", default="127.0.0.1", help="Set the host") + parser.add_argument( + "--workspace", default=None, required=True, help="Set Comfy workspace" + ) + parser.add_argument( + "--log-level", + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + help="Set the logging level", + ) + parser.add_argument( + "--comfyui-log-level", + default=None, + choices=logging._nameToLevel.keys(), + help="Set the global logging level for ComfyUI", + ) + parser.add_argument( + "--comfyui-inference-log-level", + default=None, + choices=logging._nameToLevel.keys(), + help="Set the logging level for ComfyUI inference", + ) + parser.add_argument( + "--orch-url", + default=None, + help="Orchestrator URL for capability registration", + ) + parser.add_argument( + "--orch-secret", + default=None, + help="Orchestrator secret for capability registration", + ) + parser.add_argument( + "--capability-name", + default=None, + help="Name for this capability (default: comfystream-processor)", + ) + parser.add_argument( + "--disable-frame-skip", + default=False, + action="store_true", + help="Disable adaptive frame skipping based on queue sizes (enabled by default)", + ) + parser.add_argument( + "--width", + default=512, + type=int, + help="Default video width for processing", + ) + parser.add_argument( + "--height", + default=512, + type=int, + help="Default video height for processing", + ) + args = parser.parse_args() + + logging.basicConfig( + level=args.log_level.upper(), + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%H:%M:%S", + ) + + # Allow overriding of ComfyUI log levels. + if args.comfyui_log_level: + log_level = logging._nameToLevel.get(args.comfyui_log_level.upper()) + logging.getLogger("comfy").setLevel(log_level) + + def force_print(*args, **kwargs): + print(*args, **kwargs, flush=True) + sys.stdout.flush() + + logger.info("Starting ComfyStream BYOC server with pytrickle StreamProcessor...") + + # Create frame processor with configuration + frame_processor = ComfyStreamFrameProcessor( + width=args.width, + height=args.height, + workspace=args.workspace, + disable_cuda_malloc=True, + gpu_only=True, + preview_method='none', + comfyui_inference_log_level=args.comfyui_inference_log_level + ) + + # Create frame skip configuration only if enabled + frame_skip_config = None + if args.disable_frame_skip: + logger.info("Frame skipping disabled") + else: + frame_skip_config = FrameSkipConfig() + logger.info("Frame skipping enabled: adaptive skipping based on queue sizes") + + # Create StreamProcessor with frame processor + processor = StreamProcessor( + video_processor=frame_processor.process_video_async, + audio_processor=frame_processor.process_audio_async, + model_loader=frame_processor.load_model, + param_updater=frame_processor.update_params, + on_stream_stop=frame_processor.on_stream_stop, + # Align processor name with capability for consistent logs + name=(args.capability_name or os.getenv("CAPABILITY_NAME") or "comfystream-processor"), + port=int(args.port), + host=args.host, + frame_skip_config=frame_skip_config, + # Ensure server metadata reflects the desired capability name + capability_name=(args.capability_name or os.getenv("CAPABILITY_NAME") or "comfystream-processor") + ) + + # Set the stream processor reference for text data publishing + frame_processor.set_stream_processor(processor) + + # Create async startup function to load model + async def load_model_on_startup(app): + await processor._frame_processor.load_model() + + # Create async startup function for orchestrator registration + async def register_orchestrator_startup(app): + await register_orchestrator( + orch_url=args.orch_url, + orch_secret=args.orch_secret, + capability_name=args.capability_name, + host=args.host, + port=args.port + ) + + # Add model loading and registration to startup hooks + processor.server.app.on_startup.append(load_model_on_startup) + processor.server.app.on_startup.append(register_orchestrator_startup) + + # Add warmup endpoint: accepts same body as prompts update + async def warmup_handler(request): + try: + body = await request.json() + except Exception as e: + logger.error(f"Invalid JSON in warmup request: {e}") + return web.json_response({"error": "Invalid JSON"}, status=400) + try: + # Inject sentinel to trigger warmup inside update_params on the model thread + if isinstance(body, dict): + body["warmup"] = True + else: + body = {"warmup": True} + # Fire-and-forget: do not await warmup; update_params will schedule it + asyncio.get_running_loop().create_task(frame_processor.update_params(body)) + return web.json_response({"status": "accepted"}) + except Exception as e: + logger.error(f"Warmup failed: {e}") + return web.json_response({"error": str(e)}, status=500) + + # Mount at same API namespace as StreamProcessor defaults + processor.server.add_route("POST", "/api/stream/warmup", warmup_handler) + + # Run the processor + processor.run() + + +if __name__ == "__main__": + main() diff --git a/server/frame_processor.py b/server/frame_processor.py new file mode 100644 index 00000000..bac139d4 --- /dev/null +++ b/server/frame_processor.py @@ -0,0 +1,280 @@ +import asyncio +import json +import logging +import os +from typing import List + +import numpy as np +from pytrickle.frame_processor import FrameProcessor +from pytrickle.frames import VideoFrame, AudioFrame +from comfystream.pipeline import Pipeline +from comfystream.utils import convert_prompt, ComfyStreamParamsUpdateRequest + +logger = logging.getLogger(__name__) + + +class ComfyStreamFrameProcessor(FrameProcessor): + """ + Integrated ComfyStream FrameProcessor for pytrickle. + + This class wraps the ComfyStream Pipeline to work with pytrickle's streaming architecture. + """ + + def __init__(self, text_poll_interval: float = 0.25, **load_params): + """Initialize with load parameters for pipeline creation. + + Args: + text_poll_interval: Interval in seconds to poll for text outputs (default: 0.25) + **load_params: Parameters for pipeline creation + """ + self.pipeline = None + self._load_params = load_params + self._text_poll_interval = text_poll_interval + self._stream_processor = None + self._warmup_task = None + self._text_forward_task = None + self._background_tasks = [] + self._stop_event = asyncio.Event() + super().__init__() + + def set_stream_processor(self, stream_processor): + """Set reference to StreamProcessor for data publishing.""" + self._stream_processor = stream_processor + logger.info("StreamProcessor reference set for text data publishing") + + def _setup_text_monitoring(self): + """Set up background text forwarding from the pipeline.""" + try: + if self.pipeline and self._stream_processor: + # Reset stop event for new stream + self._reset_stop_event() + # Start forwarder only if workflow has text outputs (best-effort) + should_start = True + try: + should_start = bool(self.pipeline.produces_text_output()) + except Exception: + # If capability check fails, default to starting forwarder + should_start = True + + if should_start: + # Start a background task that forwards text outputs via StreamProcessor + if self._text_forward_task and not self._text_forward_task.done(): + logger.debug("Text forwarder already running; not starting another") + return + + async def _forward_text_loop(): + try: + logger.info("Starting background text forwarder task") + while not self._stop_event.is_set(): + try: + # Non-blocking poll; sleep if no text to avoid tight loop + text = await self.pipeline.get_text_output() + if text is None or text.strip() == "": + await asyncio.sleep(self._text_poll_interval) + continue + if self._stream_processor: + success = await self._stream_processor.send_data(text) + if not success: + logger.debug("Text send failed; stopping text forwarder") + break + except asyncio.CancelledError: + logger.debug("Text forwarder task cancelled") + raise + except asyncio.CancelledError: + # Propagate to finally for cleanup + raise + except Exception as e: + logger.error(f"Error in text forwarder: {e}") + finally: + logger.info("Text forwarder task exiting") + + self._text_forward_task = asyncio.create_task(_forward_text_loop()) + self._background_tasks.append(self._text_forward_task) + except Exception: + logger.warning("Failed to set up text monitoring", exc_info=True) + + async def _stop_text_forwarder(self) -> None: + """Stop the background text forwarder task if running.""" + task = self._text_forward_task + if task and not task.done(): + try: + task.cancel() + await task + except asyncio.CancelledError: + pass + except Exception: + logger.debug("Error while awaiting text forwarder cancellation", exc_info=True) + self._text_forward_task = None + + async def on_stream_stop(self): + """Called when stream stops - cleanup background tasks.""" + logger.info("Stream stopped, cleaning up background tasks") + + # Set stop event to signal all background tasks to stop + self._stop_event.set() + + # Stop text forwarder + await self._stop_text_forwarder() + + # Cancel any other background tasks started by this processor + for task in list(self._background_tasks): + try: + if task and not task.done(): + task.cancel() + except Exception: + continue + + # Await task cancellations + for task in list(self._background_tasks): + if task: + try: + await task + except asyncio.CancelledError: + pass + except Exception: + logger.debug("Background task raised during shutdown", exc_info=True) + + self._background_tasks.clear() + logger.info("All background tasks cleaned up") + + def _reset_stop_event(self): + """Reset the stop event for a new stream.""" + self._stop_event.clear() + + async def load_model(self, **kwargs): + """Load model and initialize the pipeline.""" + params = {**self._load_params, **kwargs} + + if self.pipeline is None: + self.pipeline = Pipeline( + width=int(params.get('width', 512)), + height=int(params.get('height', 512)), + cwd=params.get('workspace', os.getcwd()), + disable_cuda_malloc=params.get('disable_cuda_malloc', True), + gpu_only=params.get('gpu_only', True), + preview_method=params.get('preview_method', 'none'), + comfyui_inference_log_level=params.get('comfyui_inference_log_level'), + ) + + async def warmup(self): + """Public warmup method that triggers pipeline warmup.""" + if not self.pipeline: + logger.warning("Warmup requested before pipeline initialization") + return + + logger.info("Running pipeline warmup...") + """Run pipeline warmup.""" + try: + capabilities = self.pipeline.get_workflow_io_capabilities() + logger.info(f"Detected I/O capabilities for warmup: {capabilities}") + + # Warm video if there are video inputs or outputs + if capabilities.get("video", {}).get("input") or capabilities.get("video", {}).get("output"): + logger.info("Running video warmup...") + await self.pipeline.warm_video() + logger.info("Video warmup completed") + + # Warm audio if there are audio inputs or outputs + if capabilities.get("audio", {}).get("input") or capabilities.get("audio", {}).get("output"): + logger.info("Running audio warmup...") + await self.pipeline.warm_audio() + logger.info("Audio warmup completed") + + except Exception as e: + logger.error(f"Warmup failed: {e}") + + def _schedule_warmup(self) -> None: + """Schedule warmup in background if not already running.""" + try: + if self._warmup_task and not self._warmup_task.done(): + logger.info("Warmup already in progress, skipping new warmup request") + return + + self._warmup_task = asyncio.create_task(self.warmup()) + logger.info("Warmup scheduled in background") + except Exception: + logger.warning("Failed to schedule warmup", exc_info=True) + + async def process_video_async(self, frame: VideoFrame) -> VideoFrame: + """Process video frame through ComfyStream Pipeline.""" + try: + + # Convert pytrickle VideoFrame to av.VideoFrame + av_frame = frame.to_av_frame(frame.tensor) + av_frame.pts = frame.timestamp + av_frame.time_base = frame.time_base + + # Process through pipeline + await self.pipeline.put_video_frame(av_frame) + processed_av_frame = await self.pipeline.get_processed_video_frame() + + # Convert back to pytrickle VideoFrame + processed_frame = VideoFrame.from_av_frame_with_timing(processed_av_frame, frame) + return processed_frame + + except Exception as e: + logger.error(f"Video processing failed: {e}") + return frame + + async def process_audio_async(self, frame: AudioFrame) -> List[AudioFrame]: + """Process audio frame through ComfyStream Pipeline or passthrough.""" + try: + if not self.pipeline: + return [frame] + + # Audio processing needed - use pipeline + av_frame = frame.to_av_frame() + await self.pipeline.put_audio_frame(av_frame) + processed_av_frame = await self.pipeline.get_processed_audio_frame() + processed_frame = AudioFrame.from_av_audio(processed_av_frame) + return [processed_frame] + + except Exception as e: + logger.error(f"Audio processing failed: {e}") + return [frame] + + async def update_params(self, params: dict): + """Update processing parameters.""" + if not self.pipeline: + return + + # Handle list input - take first element + if isinstance(params, list) and params: + params = params[0] + + # Validate parameters using the centralized validation + validated = ComfyStreamParamsUpdateRequest(**params).model_dump() + logger.info(f"Parameter validation successful, keys: {list(validated.keys())}") + + # Process prompts if provided + if "prompts" in validated and validated["prompts"]: + await self._process_prompts(validated["prompts"]) + + # Update pipeline dimensions + if "width" in validated: + self.pipeline.width = int(validated["width"]) + if "height" in validated: + self.pipeline.height = int(validated["height"]) + + # Schedule warmup if requested + if validated.get("warmup", False): + self._schedule_warmup() + + + async def _process_prompts(self, prompts): + """Process and set prompts in the pipeline.""" + try: + converted = convert_prompt(prompts, return_dict=True) + + # Set prompts in pipeline + await self.pipeline.set_prompts([converted]) + logger.info(f"Prompts set successfully: {list(prompts.keys())}") + + # Update text monitoring based on workflow capabilities + if self.pipeline.produces_text_output(): + self._setup_text_monitoring() + else: + await self._stop_text_forwarder() + + except Exception as e: + logger.error(f"Failed to process prompts: {e}") diff --git a/src/comfystream/client.py b/src/comfystream/client.py index 5c440894..b5c7dca7 100644 --- a/src/comfystream/client.py +++ b/src/comfystream/client.py @@ -23,10 +23,24 @@ def __init__(self, max_workers: int = 1, **kwargs): self._stop_event = asyncio.Event() async def set_prompts(self, prompts: List[PromptDictInput]): + """Set new prompts, replacing any existing ones. + + Args: + prompts: List of prompt dictionaries to set + + Raises: + ValueError: If prompts list is empty + Exception: If prompt conversion or validation fails + """ + if not prompts: + raise ValueError("Cannot set empty prompts list") + + # Cancel existing prompts first to avoid conflicts await self.cancel_running_prompts() # Reset stop event for new prompts self._stop_event.clear() self.current_prompts = [convert_prompt(prompt) for prompt in prompts] + logger.info(f"Queuing {len(self.current_prompts)} prompt(s) for execution") for idx in range(len(self.current_prompts)): task = asyncio.create_task(self.run_prompt(idx)) self.running_prompts[idx] = task @@ -117,7 +131,15 @@ async def get_audio_output(self): return await tensor_cache.audio_outputs.get() async def get_text_output(self): - return await tensor_cache.text_outputs.get() + try: + return tensor_cache.text_outputs.get_nowait() + except asyncio.QueueEmpty: + # Expected case - queue is empty, no text available + return None + except Exception as e: + # Unexpected errors logged for debugging + logger.warning(f"Unexpected error in get_text_output: {e}") + return None async def get_available_nodes(self): """Get metadata and available nodes info in a single pass""" diff --git a/src/comfystream/pipeline.py b/src/comfystream/pipeline.py index d7c47443..3e4febaf 100644 --- a/src/comfystream/pipeline.py +++ b/src/comfystream/pipeline.py @@ -79,7 +79,7 @@ async def warm_audio(self): return dummy_frame = av.AudioFrame() - dummy_frame.side_data.input = np.random.randint(-32768, 32767, int(48000 * 0.5), dtype=np.int16) # TODO: adds a lot of delay if it doesn't match the buffer size, is warmup needed? + dummy_frame.side_data.input = np.random.randint(-32768, 32768, int(48000 * 0.5), dtype=np.int16) dummy_frame.sample_rate = 48000 for _ in range(WARMUP_RUNS): @@ -144,7 +144,7 @@ async def put_video_frame(self, frame: av.VideoFrame): self.client.put_video_input(frame) await self.video_incoming_frames.put(frame) - async def put_audio_frame(self, frame: av.AudioFrame): + async def put_audio_frame(self, frame: av.AudioFrame, preprocess: bool = True): """Queue an audio frame for processing. Args: @@ -159,14 +159,14 @@ async def put_audio_frame(self, frame: av.AudioFrame): return # Process and send to client when input is accepted - frame.side_data.input = self.audio_preprocess(frame) + frame.side_data.input = self.audio_preprocess(frame) if preprocess else frame.to_ndarray() frame.side_data.skipped = True # Mark passthrough based on whether workflow produces audio output frame.side_data.passthrough = not self.produces_audio_output() self.client.put_audio_input(frame) await self.audio_incoming_frames.put(frame) - def video_preprocess(self, frame: av.VideoFrame) -> Union[torch.Tensor, np.ndarray]: + def video_preprocess(self, frame: av.VideoFrame) -> torch.Tensor: """Preprocess a video frame before processing. Args: @@ -178,16 +178,39 @@ def video_preprocess(self, frame: av.VideoFrame) -> Union[torch.Tensor, np.ndarr frame_np = frame.to_ndarray(format="rgb24").astype(np.float32) / 255.0 return torch.from_numpy(frame_np).unsqueeze(0) - def audio_preprocess(self, frame: av.AudioFrame) -> Union[torch.Tensor, np.ndarray]: + def audio_preprocess(self, frame: av.AudioFrame) -> np.ndarray: """Preprocess an audio frame before processing. Args: frame: The audio frame to preprocess Returns: - The preprocessed frame as a tensor or numpy array + The preprocessed frame as a numpy array with int16 dtype """ - return frame.to_ndarray().ravel().reshape(-1, 2).mean(axis=1).astype(np.int16) + audio_data = frame.to_ndarray() + + # Handle multi-dimensional audio data + if audio_data.ndim == 2 and audio_data.shape[0] == 1 and audio_data.shape[0] <= audio_data.shape[1]: + audio_data = audio_data.ravel().reshape(-1, 2).mean(axis=1) + elif audio_data.ndim > 1: + audio_data = audio_data.mean(axis=0) + + # Ensure we always return int16 data + if audio_data.dtype in [np.float32, np.float64]: + # Check if data is normalized (-1.0 to 1.0 range) + max_abs_val = np.abs(audio_data).max() + if max_abs_val <= 1.0: + # Normalized float input - scale to int16 range + audio_data = np.clip(audio_data, -1.0, 1.0) + audio_data = (audio_data * 32767).astype(np.int16) + else: + # Large float values - clip and convert directly + audio_data = np.clip(audio_data, -32768, 32767).astype(np.int16) + else: + # Already integer data - ensure it's int16 + audio_data = audio_data.astype(np.int16) + + return audio_data def video_postprocess(self, output: Union[torch.Tensor, np.ndarray]) -> av.VideoFrame: """Postprocess a video frame after processing. @@ -280,7 +303,7 @@ async def get_processed_audio_frame(self) -> av.AudioFrame: return processed_frame - async def get_text_output(self) -> str: + async def get_text_output(self) -> str | None: """Get the next text output from the pipeline. Returns: @@ -288,10 +311,11 @@ async def get_text_output(self) -> str: """ # If workflow doesn't produce text output, return empty string immediately if not self.produces_text_output(): - return "" + return None async with temporary_log_level("comfy", self._comfyui_inference_log_level): out_text = await self.client.get_text_output() + return out_text async def get_nodes_info(self) -> Dict[str, Any]: diff --git a/src/comfystream/utils.py b/src/comfystream/utils.py index e26b963d..7d8800c4 100644 --- a/src/comfystream/utils.py +++ b/src/comfystream/utils.py @@ -1,6 +1,10 @@ import copy +import json +import os +import logging import importlib -from typing import Dict, Any +from typing import Dict, Any, List, Tuple, Optional, Union +from pytrickle.api import StreamParamsUpdateRequest from comfy.api.components.schema.prompt import Prompt, PromptDictInput from .modalities import ( get_node_counts_by_type, @@ -38,7 +42,7 @@ def _validate_prompt_constraints(counts: Dict[str, int]) -> None: if counts["outputs"] == 0: raise Exception("missing output") -def convert_prompt(prompt: PromptDictInput) -> Prompt: +def convert_prompt(prompt: PromptDictInput, return_dict: bool = False) -> Prompt: """Convert a prompt by replacing specific node types with tensor equivalents.""" try: # Note: lazy import is necessary to prevent KeyError during validation @@ -46,7 +50,7 @@ def convert_prompt(prompt: PromptDictInput) -> Prompt: except Exception: pass - # Validate the schema + """Convert and validate a ComfyUI workflow prompt.""" Prompt.validate(prompt) prompt = copy.deepcopy(prompt) @@ -70,6 +74,76 @@ def convert_prompt(prompt: PromptDictInput) -> Prompt: node = prompt[key] prompt[key] = create_save_tensor_node(node["inputs"]) - # Validate the processed prompt - prompt = Prompt.validate(prompt) - return prompt + # Return dict if requested (for downstream components that expect plain dicts) + if return_dict: + return prompt # Already a plain dict at this point + + # Validate the processed prompt and return Pydantic object + return Prompt.validate(prompt) + +class ComfyStreamParamsUpdateRequest(StreamParamsUpdateRequest): + """ComfyStream parameter validation.""" + + def __init__(self, **data): + # Handle prompts parameter + if "prompts" in data: + prompts = data["prompts"] + + # Parse JSON string if needed + if isinstance(prompts, str) and prompts.strip(): + try: + prompts = json.loads(prompts) + except json.JSONDecodeError: + data.pop("prompts") + + # Handle list - use first valid dict + elif isinstance(prompts, list): + prompts = next((p for p in prompts if isinstance(p, dict)), None) + if not prompts: + data.pop("prompts") + + # Validate prompts + if "prompts" in data and isinstance(prompts, dict): + try: + data["prompts"] = convert_prompt(prompts, return_dict=True) + except Exception: + data.pop("prompts") + + # Call parent constructor + super().__init__(**data) + + @classmethod + def model_validate(cls, obj): + return cls(**obj) + + def model_dump(self): + return super().model_dump() + +def get_default_workflow() -> dict: + """Return the default workflow as a dictionary for warmup. + + Returns: + dict: Default workflow dictionary + """ + return { + "1": { + "inputs": { + "images": [ + "2", + 0 + ] + }, + "class_type": "SaveTensor", + "_meta": { + "title": "SaveTensor" + } + }, + "2": { + "inputs": {}, + "class_type": "LoadTensor", + "_meta": { + "title": "LoadTensor" + } + } + } + diff --git a/workflows/comfystream/audio-tensor-utils-example-api.json b/workflows/comfystream/audio-tensor-utils-example-api.json index 37609fe9..4d2e6c88 100644 --- a/workflows/comfystream/audio-tensor-utils-example-api.json +++ b/workflows/comfystream/audio-tensor-utils-example-api.json @@ -1,40 +1,36 @@ { - "1": { - "inputs": { - "buffer_size": 500.0 - }, - "class_type": "LoadAudioTensor", - "_meta": { - "title": "Load Audio Tensor" - } + "1": { + "inputs": { + "buffer_size": 500.0 }, - "2": { - "inputs": { - "audio": [ - "1", - 0 - ], - "sample_rate": [ - "1", - 1 - ], - "pitch_shift": 4.0 - }, - "class_type": "PitchShifter", - "_meta": { - "title": "Pitch Shift" - } + "class_type": "LoadAudioTensor", + "_meta": { + "title": "Load Audio Tensor" + } + }, + "2": { + "inputs": { + "pitch_shift": 4, + "audio": [ + "1", + 0 + ] + }, + "class_type": "PitchShifter", + "_meta": { + "title": "Pitch Shift" + } + }, + "3": { + "inputs": { + "audio": [ + "2", + 0 + ] }, - "3": { - "inputs": { - "audio": [ - "2", - 0 - ] - }, - "class_type": "SaveAudioTensor", - "_meta": { - "title": "Save Audio Tensor" - } + "class_type": "SaveAudioTensor", + "_meta": { + "title": "Save Audio Tensor" } + } } \ No newline at end of file diff --git a/workflows/comfystream/audio-transcription-api.json b/workflows/comfystream/audio-transcription-api.json index f80b94d7..57458d4b 100644 --- a/workflows/comfystream/audio-transcription-api.json +++ b/workflows/comfystream/audio-transcription-api.json @@ -1,10 +1,6 @@ { "10": { "inputs": { - "sample_rate": [ - "11", - 1 - ], "transcription_interval": 2, "accumulation_duration": 3, "whisper_model": "base", @@ -46,7 +42,7 @@ "title": "SRT Generator" } }, - "13": { + "26": { "inputs": { "data": [ "12", From 914fd8584f73a1d9e6d0d65bdae548d57c18e0a0 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Tue, 23 Sep 2025 11:42:38 -0400 Subject: [PATCH 04/19] refactor(workflows): remove ai-runner workflow trigger from docker.yaml (#414) --- .github/workflows/docker.yaml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 608a0edc..68f37474 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -83,28 +83,6 @@ jobs: cache-from: type=registry,ref=livepeer/comfyui-base:build-cache cache-to: type=registry,mode=max,ref=livepeer/comfyui-base:build-cache - trigger: - name: Trigger ai-runner workflow - needs: base - if: ${{ github.repository == 'livepeer/comfystream' }} - runs-on: ubuntu-latest - steps: - - name: Send workflow dispatch event to ai-runner - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.CI_GITHUB_TOKEN }} - script: | - await github.rest.actions.createWorkflowDispatch({ - owner: context.repo.owner, - repo: "ai-runner", - workflow_id: "comfyui-trigger.yaml", - ref: "main", - inputs: { - "comfyui-base-digest": "${{ needs.base.outputs.image-digest }}", - "triggering-branch": "${{ github.head_ref || github.ref_name }}", - }, - }); - comfystream: name: comfystream image needs: base From 4f486957feea087ecc0652d93fcaccd9d4782e3e Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Tue, 23 Sep 2025 16:42:32 -0400 Subject: [PATCH 05/19] fix(nodes): add input timeout to LoadTensor and LoadAudioTensor nodes (#423) * nodes(video): add input timeout to LoadTensor and SaveTensor nodes --- nodes/audio_utils/load_audio_tensor.py | 79 ++++++++++++++---------- nodes/tensor_utils/load_tensor.py | 33 +++++++--- server/app.py | 6 +- server/byoc.py | 5 ++ server/frame_processor.py | 13 +--- src/comfystream/__init__.py | 5 +- src/comfystream/client.py | 6 +- src/comfystream/exceptions.py | 20 ++++++ src/comfystream/server/utils/__init__.py | 2 +- src/comfystream/server/utils/utils.py | 39 ++++++++++++ src/comfystream/utils.py | 1 + 11 files changed, 156 insertions(+), 53 deletions(-) create mode 100644 src/comfystream/exceptions.py diff --git a/nodes/audio_utils/load_audio_tensor.py b/nodes/audio_utils/load_audio_tensor.py index ece7fca3..eed09eea 100644 --- a/nodes/audio_utils/load_audio_tensor.py +++ b/nodes/audio_utils/load_audio_tensor.py @@ -1,71 +1,88 @@ import numpy as np import torch - +import queue from comfystream import tensor_cache +from comfystream.exceptions import ComfyStreamInputTimeoutError, ComfyStreamAudioBufferError + class LoadAudioTensor: - CATEGORY = "audio_utils" + CATEGORY = "ComfyStream/Loaders" RETURN_TYPES = ("AUDIO",) RETURN_NAMES = ("audio",) FUNCTION = "execute" + DESCRIPTION = "Load audio tensor from ComfyStream input with timeout." def __init__(self): self.audio_buffer = np.empty(0, dtype=np.int16) self.buffer_samples = None self.sample_rate = None + self.leftover = np.empty(0, dtype=np.int16) @classmethod - def INPUT_TYPES(s): + def INPUT_TYPES(cls): return { "required": { - "buffer_size": ("FLOAT", {"default": 500.0}), + "buffer_size": ("FLOAT", { + "default": 500.0, + "tooltip": "Audio buffer size in milliseconds" + }), + }, + "optional": { + "timeout_seconds": ("FLOAT", { + "default": 1.0, + "min": 0.1, + "max": 30.0, + "step": 0.1, + "tooltip": "Timeout in seconds" + }), } } @classmethod - def IS_CHANGED(**kwargs): + def IS_CHANGED(cls, **kwargs): return float("nan") - - def execute(self, buffer_size): + + def execute(self, buffer_size: float, timeout_seconds: float = 1.0): + # Initialize if needed if self.sample_rate is None or self.buffer_samples is None: - frame = tensor_cache.audio_inputs.get(block=True) - self.sample_rate = frame.sample_rate - self.buffer_samples = int(self.sample_rate * buffer_size / 1000) - self.leftover = frame.side_data.input + try: + frame = tensor_cache.audio_inputs.get(block=True, timeout=timeout_seconds) + self.sample_rate = frame.sample_rate + self.buffer_samples = int(self.sample_rate * buffer_size / 1000) + self.leftover = frame.side_data.input + except queue.Empty: + raise ComfyStreamInputTimeoutError("audio", timeout_seconds) - if self.leftover.shape[0] < self.buffer_samples: + # Use leftover data if available + if self.leftover.shape[0] >= self.buffer_samples: + buffered_audio = self.leftover[:self.buffer_samples] + self.leftover = self.leftover[self.buffer_samples:] + else: + # Collect more audio chunks chunks = [self.leftover] if self.leftover.size > 0 else [] total_samples = self.leftover.shape[0] while total_samples < self.buffer_samples: - frame = tensor_cache.audio_inputs.get(block=True) - if frame.sample_rate != self.sample_rate: - raise ValueError("Sample rate mismatch") - chunks.append(frame.side_data.input) - total_samples += frame.side_data.input.shape[0] + try: + frame = tensor_cache.audio_inputs.get(block=True, timeout=timeout_seconds) + if frame.sample_rate != self.sample_rate: + raise ValueError(f"Sample rate mismatch: expected {self.sample_rate}Hz, got {frame.sample_rate}Hz") + chunks.append(frame.side_data.input) + total_samples += frame.side_data.input.shape[0] + except queue.Empty: + raise ComfyStreamAudioBufferError(timeout_seconds, self.buffer_samples, total_samples) merged_audio = np.concatenate(chunks, dtype=np.int16) buffered_audio = merged_audio[:self.buffer_samples] - self.leftover = merged_audio[self.buffer_samples:] - else: - buffered_audio = self.leftover[:self.buffer_samples] - self.leftover = self.leftover[self.buffer_samples:] + self.leftover = merged_audio[self.buffer_samples:] if merged_audio.shape[0] > self.buffer_samples else np.empty(0, dtype=np.int16) - # Convert numpy array to torch tensor and normalize int16 to float32 + # Convert to ComfyUI AUDIO format waveform_tensor = torch.from_numpy(buffered_audio.astype(np.float32) / 32768.0) # Ensure proper tensor shape: (batch, channels, samples) if waveform_tensor.dim() == 1: - # Mono: (samples,) -> (1, 1, samples) waveform_tensor = waveform_tensor.unsqueeze(0).unsqueeze(0) elif waveform_tensor.dim() == 2: - # Assume (channels, samples) and add batch dimension waveform_tensor = waveform_tensor.unsqueeze(0) - # Return AUDIO dictionary format - audio_dict = { - "waveform": waveform_tensor, - "sample_rate": self.sample_rate - } - - return (audio_dict,) + return ({"waveform": waveform_tensor, "sample_rate": self.sample_rate},) \ No newline at end of file diff --git a/nodes/tensor_utils/load_tensor.py b/nodes/tensor_utils/load_tensor.py index c39fe8a1..9923a996 100644 --- a/nodes/tensor_utils/load_tensor.py +++ b/nodes/tensor_utils/load_tensor.py @@ -1,20 +1,37 @@ +import torch +import queue from comfystream import tensor_cache +from comfystream.exceptions import ComfyStreamInputTimeoutError class LoadTensor: - CATEGORY = "tensor_utils" + CATEGORY = "ComfyStream/Loaders" RETURN_TYPES = ("IMAGE",) FUNCTION = "execute" + DESCRIPTION = "Load image tensor from ComfyStream input with timeout." @classmethod - def INPUT_TYPES(s): - return {} + def INPUT_TYPES(cls): + return { + "optional": { + "timeout_seconds": ("FLOAT", { + "default": 1.0, + "min": 0.1, + "max": 30.0, + "step": 0.1, + "tooltip": "Timeout in seconds" + }), + } + } @classmethod - def IS_CHANGED(): + def IS_CHANGED(cls, **kwargs): return float("nan") - def execute(self): - frame = tensor_cache.image_inputs.get(block=True) - frame.side_data.skipped = False - return (frame.side_data.input,) + def execute(self, timeout_seconds: float = 1.0): + try: + frame = tensor_cache.image_inputs.get(block=True, timeout=timeout_seconds) + frame.side_data.skipped = False + return (frame.side_data.input,) + except queue.Empty: + raise ComfyStreamInputTimeoutError("video", timeout_seconds) diff --git a/server/app.py b/server/app.py index a3a42fc4..c0706569 100644 --- a/server/app.py +++ b/server/app.py @@ -29,7 +29,7 @@ from aiortc.rtcrtpsender import RTCRtpSender from comfystream.pipeline import Pipeline from twilio.rest import Client -from comfystream.server.utils import patch_loop_datagram, add_prefix_to_app_routes, FPSMeter +from comfystream.server.utils import patch_loop_datagram, add_prefix_to_app_routes, FPSMeter, ComfyStreamTimeoutFilter from comfystream.server.metrics import MetricsManager, StreamStatsManager import time @@ -694,6 +694,10 @@ def force_print(*args, **kwargs): if args.comfyui_log_level: log_level = logging._nameToLevel.get(args.comfyui_log_level.upper()) logging.getLogger("comfy").setLevel(log_level) + + # Add ComfyStream timeout filter to suppress verbose execution logging + logging.getLogger("comfy.cmd.execution").addFilter(ComfyStreamTimeoutFilter()) + logging.getLogger("comfy").addFilter(ComfyStreamTimeoutFilter()) if args.comfyui_inference_log_level: app["comfui_inference_log_level"] = args.comfyui_inference_log_level diff --git a/server/byoc.py b/server/byoc.py index 0735674b..19a166be 100644 --- a/server/byoc.py +++ b/server/byoc.py @@ -14,6 +14,7 @@ from pytrickle.utils.register import RegisterCapability from pytrickle.frame_skipper import FrameSkipConfig from frame_processor import ComfyStreamFrameProcessor +from comfystream.server.utils import ComfyStreamTimeoutFilter logger = logging.getLogger(__name__) @@ -117,6 +118,10 @@ def main(): if args.comfyui_log_level: log_level = logging._nameToLevel.get(args.comfyui_log_level.upper()) logging.getLogger("comfy").setLevel(log_level) + + # Add ComfyStream timeout filter to suppress verbose execution logging + logging.getLogger("comfy.cmd.execution").addFilter(ComfyStreamTimeoutFilter()) + logging.getLogger("comfy").addFilter(ComfyStreamTimeoutFilter()) def force_print(*args, **kwargs): print(*args, **kwargs, flush=True) diff --git a/server/frame_processor.py b/server/frame_processor.py index bac139d4..6cc79a71 100644 --- a/server/frame_processor.py +++ b/server/frame_processor.py @@ -157,28 +157,21 @@ async def load_model(self, **kwargs): ) async def warmup(self): - """Public warmup method that triggers pipeline warmup.""" + """Warm up the pipeline.""" if not self.pipeline: logger.warning("Warmup requested before pipeline initialization") return logger.info("Running pipeline warmup...") - """Run pipeline warmup.""" try: capabilities = self.pipeline.get_workflow_io_capabilities() - logger.info(f"Detected I/O capabilities for warmup: {capabilities}") + logger.info(f"Detected I/O capabilities: {capabilities}") - # Warm video if there are video inputs or outputs if capabilities.get("video", {}).get("input") or capabilities.get("video", {}).get("output"): - logger.info("Running video warmup...") await self.pipeline.warm_video() - logger.info("Video warmup completed") - # Warm audio if there are audio inputs or outputs if capabilities.get("audio", {}).get("input") or capabilities.get("audio", {}).get("output"): - logger.info("Running audio warmup...") await self.pipeline.warm_audio() - logger.info("Audio warmup completed") except Exception as e: logger.error(f"Warmup failed: {e}") @@ -265,7 +258,7 @@ async def _process_prompts(self, prompts): """Process and set prompts in the pipeline.""" try: converted = convert_prompt(prompts, return_dict=True) - + # Set prompts in pipeline await self.pipeline.set_prompts([converted]) logger.info(f"Prompts set successfully: {list(prompts.keys())}") diff --git a/src/comfystream/__init__.py b/src/comfystream/__init__.py index b58bf2e4..8aee624c 100644 --- a/src/comfystream/__init__.py +++ b/src/comfystream/__init__.py @@ -3,6 +3,7 @@ from .server.utils import temporary_log_level from .server.utils import FPSMeter from .server.metrics import MetricsManager, StreamStatsManager +from .exceptions import ComfyStreamInputTimeoutError, ComfyStreamAudioBufferError __all__ = [ 'ComfyStreamClient', @@ -10,5 +11,7 @@ 'temporary_log_level', 'FPSMeter', 'MetricsManager', - 'StreamStatsManager' + 'StreamStatsManager', + 'ComfyStreamInputTimeoutError', + 'ComfyStreamAudioBufferError' ] diff --git a/src/comfystream/client.py b/src/comfystream/client.py index b5c7dca7..291ecdda 100644 --- a/src/comfystream/client.py +++ b/src/comfystream/client.py @@ -4,6 +4,7 @@ from comfystream import tensor_cache from comfystream.utils import convert_prompt +from comfystream.exceptions import ComfyStreamInputTimeoutError from comfy.api.components.schema.prompt import PromptDictInput from comfy.cli_args_types import Configuration @@ -68,6 +69,9 @@ async def run_prompt(self, prompt_index: int): await self.comfy_client.queue_prompt(self.current_prompts[prompt_index]) except asyncio.CancelledError: raise + except ComfyStreamInputTimeoutError: + # Timeout errors are expected during stream switching - just continue + continue except Exception as e: await self.cleanup() logger.error(f"Error running prompt: {str(e)}") @@ -265,4 +269,4 @@ async def get_available_nodes(self): except Exception as e: logger.error(f"Error getting node info: {str(e)}") - return {} + return {} \ No newline at end of file diff --git a/src/comfystream/exceptions.py b/src/comfystream/exceptions.py new file mode 100644 index 00000000..8382a4c3 --- /dev/null +++ b/src/comfystream/exceptions.py @@ -0,0 +1,20 @@ +"""ComfyStream specific exceptions.""" + + +class ComfyStreamInputTimeoutError(Exception): + """Raised when input tensors are not available within timeout.""" + + def __init__(self, input_type: str, timeout_seconds: float): + self.input_type = input_type + self.timeout_seconds = timeout_seconds + message = f"No {input_type} frames available after {timeout_seconds}s timeout" + super().__init__(message) + + +class ComfyStreamAudioBufferError(ComfyStreamInputTimeoutError): + """Audio buffer insufficient data error.""" + + def __init__(self, timeout_seconds: float, needed_samples: int, available_samples: int): + self.needed_samples = needed_samples + self.available_samples = available_samples + super().__init__("audio", timeout_seconds) diff --git a/src/comfystream/server/utils/__init__.py b/src/comfystream/server/utils/__init__.py index daa71bb1..31a55908 100644 --- a/src/comfystream/server/utils/__init__.py +++ b/src/comfystream/server/utils/__init__.py @@ -1,2 +1,2 @@ -from .utils import patch_loop_datagram, add_prefix_to_app_routes, temporary_log_level +from .utils import patch_loop_datagram, add_prefix_to_app_routes, temporary_log_level, ComfyStreamTimeoutFilter from .fps_meter import FPSMeter diff --git a/src/comfystream/server/utils/utils.py b/src/comfystream/server/utils/utils.py index c7a7ac30..974aa74a 100644 --- a/src/comfystream/server/utils/utils.py +++ b/src/comfystream/server/utils/utils.py @@ -83,3 +83,42 @@ async def temporary_log_level(logger_name: str, level: int): finally: if level is not None: logger.setLevel(original_level) + + +class ComfyStreamTimeoutFilter(logging.Filter): + """Filter to suppress verbose ComfyUI execution logs for ComfyStream timeout exceptions.""" + + def filter(self, record): + """Filter out ComfyUI execution error logs for ComfyStream timeout exceptions.""" + # Only filter ERROR level messages from ComfyUI execution system + if record.levelno != logging.ERROR: + return True + + # Check if this is from ComfyUI execution system + if not (record.name.startswith("comfy") and ("execution" in record.name or record.name == "comfy")): + return True + + # Get the full message including any exception info + message = record.getMessage() + + # Check if this is a ComfyStream timeout-related error + timeout_indicators = [ + "ComfyStreamInputTimeoutError", + "ComfyStreamAudioBufferError", + "No video frames available", + "No audio frames available" + ] + + # Suppress if any timeout indicator is found in the message + for indicator in timeout_indicators: + if indicator in message: + return False + + # Also check the exception info if present + if record.exc_info and record.exc_info[1]: + exc_str = str(record.exc_info[1]) + for indicator in timeout_indicators: + if indicator in exc_str: + return False + + return True diff --git a/src/comfystream/utils.py b/src/comfystream/utils.py index 7d8800c4..c2cecd05 100644 --- a/src/comfystream/utils.py +++ b/src/comfystream/utils.py @@ -73,6 +73,7 @@ def convert_prompt(prompt: PromptDictInput, return_dict: bool = False) -> Prompt for key in convertible_keys["PreviewImage"] + convertible_keys["SaveImage"]: node = prompt[key] prompt[key] = create_save_tensor_node(node["inputs"]) + # Return dict if requested (for downstream components that expect plain dicts) if return_dict: From bcebd510e2d8bcf147f4d4d7b36408f7b32f61eb Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Thu, 25 Sep 2025 15:23:29 -0400 Subject: [PATCH 06/19] cleanup(nodes): Improve input exception logging (#427) * refactor(exceptions): improve timeout error logging for ComfyStreamAudioBufferError, ComfyStreamInputTimeoutError --- server/app.py | 4 +- server/byoc.py | 3 +- server/frame_processor.py | 8 ++ src/comfystream/client.py | 1 + src/comfystream/exceptions.py | 113 ++++++++++++++++++++++- src/comfystream/server/utils/__init__.py | 2 +- src/comfystream/server/utils/utils.py | 38 -------- 7 files changed, 123 insertions(+), 46 deletions(-) diff --git a/server/app.py b/server/app.py index c0706569..028f12bb 100644 --- a/server/app.py +++ b/server/app.py @@ -29,7 +29,8 @@ from aiortc.rtcrtpsender import RTCRtpSender from comfystream.pipeline import Pipeline from twilio.rest import Client -from comfystream.server.utils import patch_loop_datagram, add_prefix_to_app_routes, FPSMeter, ComfyStreamTimeoutFilter +from comfystream.server.utils import patch_loop_datagram, add_prefix_to_app_routes, FPSMeter +from comfystream.exceptions import ComfyStreamTimeoutFilter from comfystream.server.metrics import MetricsManager, StreamStatsManager import time @@ -697,7 +698,6 @@ def force_print(*args, **kwargs): # Add ComfyStream timeout filter to suppress verbose execution logging logging.getLogger("comfy.cmd.execution").addFilter(ComfyStreamTimeoutFilter()) - logging.getLogger("comfy").addFilter(ComfyStreamTimeoutFilter()) if args.comfyui_inference_log_level: app["comfui_inference_log_level"] = args.comfyui_inference_log_level diff --git a/server/byoc.py b/server/byoc.py index 19a166be..89566748 100644 --- a/server/byoc.py +++ b/server/byoc.py @@ -14,7 +14,7 @@ from pytrickle.utils.register import RegisterCapability from pytrickle.frame_skipper import FrameSkipConfig from frame_processor import ComfyStreamFrameProcessor -from comfystream.server.utils import ComfyStreamTimeoutFilter +from comfystream.exceptions import ComfyStreamTimeoutFilter logger = logging.getLogger(__name__) @@ -121,7 +121,6 @@ def main(): # Add ComfyStream timeout filter to suppress verbose execution logging logging.getLogger("comfy.cmd.execution").addFilter(ComfyStreamTimeoutFilter()) - logging.getLogger("comfy").addFilter(ComfyStreamTimeoutFilter()) def force_print(*args, **kwargs): print(*args, **kwargs, flush=True) diff --git a/server/frame_processor.py b/server/frame_processor.py index 6cc79a71..65d1c0ac 100644 --- a/server/frame_processor.py +++ b/server/frame_processor.py @@ -113,6 +113,14 @@ async def on_stream_stop(self): # Set stop event to signal all background tasks to stop self._stop_event.set() + # Stop the ComfyStream client's prompt execution + if self.pipeline and self.pipeline.client: + logger.info("Stopping ComfyStream client prompt execution") + try: + await self.pipeline.client.cleanup() + except Exception as e: + logger.error(f"Error stopping ComfyStream client: {e}") + # Stop text forwarder await self._stop_text_forwarder() diff --git a/src/comfystream/client.py b/src/comfystream/client.py index 291ecdda..7686fca1 100644 --- a/src/comfystream/client.py +++ b/src/comfystream/client.py @@ -71,6 +71,7 @@ async def run_prompt(self, prompt_index: int): raise except ComfyStreamInputTimeoutError: # Timeout errors are expected during stream switching - just continue + logger.info(f"Input for prompt {prompt_index} timed out, continuing") continue except Exception as e: await self.cleanup() diff --git a/src/comfystream/exceptions.py b/src/comfystream/exceptions.py index 8382a4c3..aaf20c84 100644 --- a/src/comfystream/exceptions.py +++ b/src/comfystream/exceptions.py @@ -1,20 +1,127 @@ """ComfyStream specific exceptions.""" +import logging +from typing import Dict, Any, Optional + + +def log_comfystream_error( + exception: Exception, + logger: Optional[logging.Logger] = None, + level: int = logging.ERROR +) -> None: + """ + Centralized logging function for ComfyStream exceptions. + + Args: + exception: The exception to log + logger: Optional logger to use (defaults to module logger) + level: Log level (defaults to ERROR) + """ + if logger is None: + logger = logging.getLogger(__name__) + + # If it's a ComfyStream timeout error with structured details, use its logging method + if isinstance(exception, ComfyStreamInputTimeoutError): + exception.log_error(logger) + else: + # For other exceptions, provide basic logging + logger.log(level, f"ComfyStream error: {type(exception).__name__}: {str(exception)}") + class ComfyStreamInputTimeoutError(Exception): """Raised when input tensors are not available within timeout.""" - def __init__(self, input_type: str, timeout_seconds: float): + def __init__( + self, + input_type: str, + timeout_seconds: float, + details: Optional[Dict[str, Any]] = None + ): self.input_type = input_type self.timeout_seconds = timeout_seconds + self.details = details or {} message = f"No {input_type} frames available after {timeout_seconds}s timeout" super().__init__(message) + + def get_log_details(self) -> Dict[str, Any]: + """Get structured details for logging.""" + base_details = { + "input_type": self.input_type, + "timeout_seconds": self.timeout_seconds + } + base_details.update(self.details) + return base_details + + def log_error(self, logger: Optional[logging.Logger] = None) -> None: + """Log the error with detailed information.""" + if logger is None: + logger = logging.getLogger(__name__) + + details = self.get_log_details() + detail_str = ", ".join(f"{k}={v}" for k, v in details.items()) + logger.error(f"ComfyStream timeout error: {str(self)} | Details: {detail_str}") class ComfyStreamAudioBufferError(ComfyStreamInputTimeoutError): """Audio buffer insufficient data error.""" - def __init__(self, timeout_seconds: float, needed_samples: int, available_samples: int): + def __init__( + self, + timeout_seconds: float, + needed_samples: int, + available_samples: int + ): self.needed_samples = needed_samples self.available_samples = available_samples - super().__init__("audio", timeout_seconds) + + # Pass audio-specific details to the base class + audio_details = { + "needed_samples": needed_samples, + "available_samples": available_samples, + } + super().__init__("audio", timeout_seconds, details=audio_details) + + def get_log_details(self) -> Dict[str, Any]: + """Get structured details for logging, with audio-specific formatting.""" + details = super().get_log_details() + return details + + +class ComfyStreamTimeoutFilter(logging.Filter): + """Filter to suppress verbose ComfyUI execution logs for ComfyStream timeout exceptions.""" + + def filter(self, record): + """Filter out ComfyUI execution error logs for ComfyStream timeout exceptions.""" + try: + # Only filter ERROR level messages from ComfyUI execution system + if record.levelno != logging.ERROR: + return True + + # Check if this is from ComfyUI execution system + if not (record.name.startswith("comfy") and ("execution" in record.name or record.name == "comfy")): + return True + + # Get the full message including any exception info + message = record.getMessage() + + # Simple check: if this log contains ComfyStreamAudioBufferError or ComfyStreamInputTimeoutError, suppress it + if ("ComfyStreamAudioBufferError" in message or + "ComfyStreamInputTimeoutError" in message): + return False + + # Also check the exception info if present + if record.exc_info and record.exc_info[1]: + exc_str = str(record.exc_info[1]) + exc_type = str(type(record.exc_info[1])) + + if ("ComfyStreamAudioBufferError" in exc_str or + "ComfyStreamInputTimeoutError" in exc_str or + "ComfyStreamAudioBufferError" in exc_type or + "ComfyStreamInputTimeoutError" in exc_type): + return False + + return True + except Exception as e: + # If filter fails, allow the log through and print the error + print(f"[FILTER ERROR] Filter failed: {e}") + return True diff --git a/src/comfystream/server/utils/__init__.py b/src/comfystream/server/utils/__init__.py index 31a55908..daa71bb1 100644 --- a/src/comfystream/server/utils/__init__.py +++ b/src/comfystream/server/utils/__init__.py @@ -1,2 +1,2 @@ -from .utils import patch_loop_datagram, add_prefix_to_app_routes, temporary_log_level, ComfyStreamTimeoutFilter +from .utils import patch_loop_datagram, add_prefix_to_app_routes, temporary_log_level from .fps_meter import FPSMeter diff --git a/src/comfystream/server/utils/utils.py b/src/comfystream/server/utils/utils.py index 974aa74a..96e6661e 100644 --- a/src/comfystream/server/utils/utils.py +++ b/src/comfystream/server/utils/utils.py @@ -84,41 +84,3 @@ async def temporary_log_level(logger_name: str, level: int): if level is not None: logger.setLevel(original_level) - -class ComfyStreamTimeoutFilter(logging.Filter): - """Filter to suppress verbose ComfyUI execution logs for ComfyStream timeout exceptions.""" - - def filter(self, record): - """Filter out ComfyUI execution error logs for ComfyStream timeout exceptions.""" - # Only filter ERROR level messages from ComfyUI execution system - if record.levelno != logging.ERROR: - return True - - # Check if this is from ComfyUI execution system - if not (record.name.startswith("comfy") and ("execution" in record.name or record.name == "comfy")): - return True - - # Get the full message including any exception info - message = record.getMessage() - - # Check if this is a ComfyStream timeout-related error - timeout_indicators = [ - "ComfyStreamInputTimeoutError", - "ComfyStreamAudioBufferError", - "No video frames available", - "No audio frames available" - ] - - # Suppress if any timeout indicator is found in the message - for indicator in timeout_indicators: - if indicator in message: - return False - - # Also check the exception info if present - if record.exc_info and record.exc_info[1]: - exc_str = str(record.exc_info[1]) - for indicator in timeout_indicators: - if indicator in exc_str: - return False - - return True From 5c66950c44261572fc04f9af354e236ea9527357 Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Fri, 26 Sep 2025 12:59:05 -0400 Subject: [PATCH 07/19] Update documentation links to docs.comfystream.org (#431) Co-authored-by: Cursor Agent --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 52c864f1..a9edf4be 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ This repo also includes a WebRTC server and UI that uses comfystream to support Refer to [.devcontainer/README.md](.devcontainer/README.md) to setup ComfyStream in a devcontainer using a pre-configured ComfyUI docker environment. -For other installation options, refer to [Install ComfyUI and ComfyStream](https://pipelines.livepeer.org/docs/technical/install/local-testing) in the Livepeer pipelines documentation. +For other installation options, refer to [Install ComfyUI and ComfyStream](https://docs.comfystream.org/technical/get-started/install) in the ComfyStream documentation. For additional information, refer to the remaining sections below. @@ -35,7 +35,7 @@ For additional information, refer to the remaining sections below. You can quickly deploy ComfyStream using the docker image `livepeer/comfystream` -Refer to the documentation at [https://pipelines.livepeer.org/docs/technical/getting-started/install-comfystream](https://pipelines.livepeer.org/docs/technical/getting-started/install-comfystream) for instructions to run locally or on a remote server. +Refer to the documentation at [https://docs.comfystream.org/technical/get-started/install](https://docs.comfystream.org/technical/get-started/install) for instructions to run locally or on a remote server. #### RunPod From e25d25a19659ce4d5f1a5cb882e3abb1cf7b762d Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Fri, 26 Sep 2025 14:24:43 -0400 Subject: [PATCH 08/19] launch.json: update capability name for byoc (#437) --- .vscode/launch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 4d442c58..f05e02f5 100755 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -56,7 +56,7 @@ "env": { "ORCH_URL": "https://172.17.0.1:9995", "ORCH_SECRET": "orch-secret", - "CAPABILITY_NAME": "comfystream-byoc-processor", + "CAPABILITY_NAME": "comfystream", "CAPABILITY_DESCRIPTION": "ComfyUI streaming processor for BYOC mode", "CAPABILITY_URL": "http://172.17.0.1:8000", "CAPABILITY_PRICE_PER_UNIT": "0", From 29a51d5592624c6ae2d1c8a75fde1517f4ba653a Mon Sep 17 00:00:00 2001 From: John | Elite Encoder Date: Fri, 26 Sep 2025 15:15:54 -0400 Subject: [PATCH 09/19] fix(docker): add missing development libraries for Cairo and Pango (#438) Co-authored-by: Jason Stone --- docker/Dockerfile.base | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index c8f6b7ff..7d3facda 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -27,6 +27,12 @@ RUN apt update && apt install -yqq --no-install-recommends \ swig \ libprotobuf-dev \ protobuf-compiler \ + libcairo2-dev \ + libpango1.0-dev \ + libgdk-pixbuf2.0-dev \ + libffi-dev \ + libgirepository1.0-dev \ + pkg-config \ && rm -rf /var/lib/apt/lists/* #enable opengl support with nvidia gpu From 70f924ca702db8f7d64fc889c357c1373e7bc5af Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Sat, 27 Sep 2025 06:40:27 +0300 Subject: [PATCH 10/19] feat: GitHub workflow opencv-cuda compilation (#432) * feat: GitHub workflow for automated OpenCV CUDA builds --- .github/workflows/opencv-cuda-artifact.yml | 196 +++++++++++++++++++++ docker/Dockerfile.opencv | 124 +++++++++++++ 2 files changed, 320 insertions(+) create mode 100644 .github/workflows/opencv-cuda-artifact.yml create mode 100644 docker/Dockerfile.opencv diff --git a/.github/workflows/opencv-cuda-artifact.yml b/.github/workflows/opencv-cuda-artifact.yml new file mode 100644 index 00000000..5572d09e --- /dev/null +++ b/.github/workflows/opencv-cuda-artifact.yml @@ -0,0 +1,196 @@ +name: Build OpenCV CUDA Artifact + +on: + push: + branches: + - main + paths: + - 'docker/Dockerfile.opencv' + - 'docker/Dockerfile.base' + pull_request: + branches: + - main + paths: + - 'docker/Dockerfile.opencv' + - 'docker/Dockerfile.base' + workflow_dispatch: + inputs: + python_version: + description: 'Python version to build' + required: false + default: '3.12' + type: string + cuda_version: + description: 'CUDA version to build' + required: false + default: '12.8' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + PYTHON_VERSION: ${{ github.event.inputs.python_version || '3.12' }} + CUDA_VERSION: ${{ github.event.inputs.cuda_version || '12.8' }} + +jobs: + build-opencv-artifact: + name: Build OpenCV CUDA Artifact + runs-on: [self-hosted, linux, gpu] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build OpenCV CUDA Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile.opencv + build-args: | + BASE_IMAGE=nvidia/cuda:${{ env.CUDA_VERSION }}.1-cudnn-devel-ubuntu22.04 + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + CUDA_VERSION=${{ env.CUDA_VERSION }} + tags: opencv-cuda-artifact:latest + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Extract OpenCV libraries from Docker container + run: | + echo "Creating temporary container..." + docker create --name opencv-extract opencv-cuda-artifact:latest + + echo "Creating workspace directory..." + mkdir -p ./opencv-artifacts + + # Try to copy from system installation + docker cp opencv-extract:/usr/local/lib/python${{ env.PYTHON_VERSION }}/site-packages/cv2 ./opencv-artifacts/cv2 || echo "cv2 not found in system site-packages" + + echo "Copying OpenCV source directories..." + # Copy opencv and opencv_contrib source directories + docker cp opencv-extract:/workspace/opencv ./opencv-artifacts/ || echo "opencv source not found" + docker cp opencv-extract:/workspace/opencv_contrib ./opencv-artifacts/ || echo "opencv_contrib source not found" + + echo "Cleaning up container..." + docker rm opencv-extract + + echo "Contents of opencv-artifacts:" + ls -la ./opencv-artifacts/ + + - name: Create tarball artifact + run: | + echo "Creating opencv-cuda-release.tar.gz..." + cd ./opencv-artifacts + tar -czf ../opencv-cuda-release.tar.gz . || echo "Failed to create tarball" + cd .. + + echo "Generating checksums..." + sha256sum opencv-cuda-release.tar.gz > opencv-cuda-release.tar.gz.sha256 + md5sum opencv-cuda-release.tar.gz > opencv-cuda-release.tar.gz.md5 + + echo "Verifying archive contents..." + echo "Archive size: $(ls -lh opencv-cuda-release.tar.gz | awk '{print $5}')" + echo "First 20 files in archive:" + tar -tzf opencv-cuda-release.tar.gz | head -20 + + - name: Extract and verify tarball + run: | + echo "Testing tarball extraction..." + mkdir -p test-extract + cd test-extract + tar -xzf ../opencv-cuda-release.tar.gz + echo "Extracted contents:" + find . -maxdepth 2 -type d | sort + cd .. + rm -rf test-extract + + - name: Upload OpenCV CUDA Release Artifact + uses: actions/upload-artifact@v4 + with: + name: opencv-cuda-release-python${{ env.PYTHON_VERSION }}-cuda${{ env.CUDA_VERSION }}-${{ github.sha }} + path: | + opencv-cuda-release.tar.gz + opencv-cuda-release.tar.gz.sha256 + opencv-cuda-release.tar.gz.md5 + retention-days: 30 + + - name: Create Release Notes + run: | + cat > release-info.txt << EOF + OpenCV CUDA Release Artifact + + Build Details: + - Python Version: ${{ env.PYTHON_VERSION }} + - CUDA Version: ${{ env.CUDA_VERSION }} + - OpenCV Version: 4.11.0 + - Built on: $(date -u) + - Commit SHA: ${{ github.sha }} + + Contents: + - cv2: Python OpenCV module with CUDA support + - opencv: OpenCV source code + - opencv_contrib: OpenCV contrib modules source + - lib: Compiled OpenCV libraries + - include: OpenCV header files + + Installation: + 1. Download opencv-cuda-release.tar.gz + 2. Extract: tar -xzf opencv-cuda-release.tar.gz + 3. Copy cv2 to your Python environment's site-packages + 4. Ensure CUDA libraries are in your system PATH + + Checksums: + SHA256: $(cat opencv-cuda-release.tar.gz.sha256) + MD5: $(cat opencv-cuda-release.tar.gz.md5) + EOF + + - name: Upload Release Info + uses: actions/upload-artifact@v4 + with: + name: release-info-python${{ env.PYTHON_VERSION }}-cuda${{ env.CUDA_VERSION }}-${{ github.sha }} + path: release-info.txt + retention-days: 30 + + create-release-draft: + name: Create Release Draft + needs: build-opencv-artifact + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: opencv-cuda-release-python${{ env.PYTHON_VERSION }}-cuda${{ env.CUDA_VERSION }}-${{ github.sha }} + path: ./artifacts + + - name: Download release info + uses: actions/download-artifact@v4 + with: + name: release-info-python${{ env.PYTHON_VERSION }}-cuda${{ env.CUDA_VERSION }}-${{ github.sha }} + path: ./artifacts + + - name: Create Release Draft + uses: softprops/action-gh-release@v1 + with: + tag_name: opencv-cuda-v${{ env.PYTHON_VERSION }}-${{ env.CUDA_VERSION }}-${{ github.run_number }} + name: OpenCV CUDA Release - Python ${{ env.PYTHON_VERSION }} CUDA ${{ env.CUDA_VERSION }} + body_path: ./artifacts/release-info.txt + draft: true + files: | + ./artifacts/opencv-cuda-release.tar.gz + ./artifacts/opencv-cuda-release.tar.gz.sha256 + ./artifacts/opencv-cuda-release.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/docker/Dockerfile.opencv b/docker/Dockerfile.opencv new file mode 100644 index 00000000..848db1fe --- /dev/null +++ b/docker/Dockerfile.opencv @@ -0,0 +1,124 @@ +ARG BASE_IMAGE=nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04 \ + CONDA_VERSION=latest \ + PYTHON_VERSION=3.12 \ + CUDA_VERSION=12.8 + +FROM "${BASE_IMAGE}" + +ARG CONDA_VERSION \ + PYTHON_VERSION \ + CUDA_VERSION + +ENV DEBIAN_FRONTEND=noninteractive \ + CONDA_VERSION="${CONDA_VERSION}" \ + PATH="/workspace/miniconda3/bin:${PATH}" \ + PYTHON_VERSION="${PYTHON_VERSION}" \ + CUDA_VERSION="${CUDA_VERSION}" + +# System dependencies +RUN apt update && apt install -yqq \ + git \ + wget \ + nano \ + socat \ + libsndfile1 \ + build-essential \ + llvm \ + tk-dev \ + cmake \ + libgflags-dev \ + libgoogle-glog-dev \ + libjpeg-dev \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ + libswscale-dev && \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /workspace/comfystream && \ + wget "https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh" -O /tmp/miniconda.sh && \ + bash /tmp/miniconda.sh -b -p /workspace/miniconda3 && \ + eval "$(/workspace/miniconda3/bin/conda shell.bash hook)" && \ + conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main && \ + conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r && \ + conda create -n comfystream python="${PYTHON_VERSION}" -c conda-forge -y && \ + rm /tmp/miniconda.sh && \ + conda run -n comfystream --no-capture-output pip install numpy==1.26.4 aiortc aiohttp requests tqdm pyyaml --root-user-action=ignore + +# Clone ComfyUI +ADD --link https://github.com/comfyanonymous/ComfyUI.git /workspace/ComfyUI + +# OpenCV with CUDA support +WORKDIR /workspace + +# Clone OpenCV repositories +RUN git clone --depth 1 --branch 4.11.0 https://github.com/opencv/opencv.git && \ + git clone --depth 1 --branch 4.11.0 https://github.com/opencv/opencv_contrib.git + +# Create build directory +RUN mkdir -p /workspace/opencv/build + +# Create a toolchain file with absolute path +RUN echo '# Custom toolchain file to exclude Conda paths\n\ +\n\ +# Set system compilers\n\ +set(CMAKE_C_COMPILER "/usr/bin/gcc")\n\ +set(CMAKE_CXX_COMPILER "/usr/bin/g++")\n\ +\n\ +# Set system root directories\n\ +set(CMAKE_FIND_ROOT_PATH "/usr")\n\ +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)\n\ +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\n\ +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\n\ +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)\n\ +\n\ +# Explicitly exclude Conda paths\n\ +list(APPEND CMAKE_IGNORE_PATH \n\ + "/workspace/miniconda3"\n\ + "/workspace/miniconda3/envs"\n\ + "/workspace/miniconda3/envs/comfystream"\n\ + "/workspace/miniconda3/envs/comfystream/lib"\n\ +)\n\ +\n\ +# Set RPATH settings\n\ +set(CMAKE_SKIP_BUILD_RPATH FALSE)\n\ +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)\n\ +set(CMAKE_INSTALL_RPATH "/usr/local/lib:/usr/lib/x86_64-linux-gnu")\n\ +set(PYTHON_LIBRARY "/workspace/miniconda3/envs/comfystream/lib/")\n\ +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)' > /workspace/custom_toolchain.cmake + +# Set environment variables for OpenCV +RUN echo 'export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH' >> /root/.bashrc + +# Build and install OpenCV with CUDA support +RUN cd /workspace/opencv/build && \ + # Build OpenCV + cmake \ + -D CMAKE_TOOLCHAIN_FILE=/workspace/custom_toolchain.cmake \ + -D CMAKE_BUILD_TYPE=RELEASE \ + -D CMAKE_INSTALL_PREFIX=/usr/local \ + -D WITH_CUDA=ON \ + -D WITH_CUDNN=ON \ + -D WITH_CUBLAS=ON \ + -D WITH_TBB=ON \ + -D CUDA_ARCH_LIST="8.0+PTX" \ + -D OPENCV_DNN_CUDA=ON \ + -D OPENCV_ENABLE_NONFREE=ON \ + -D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \ + -D OPENCV_EXTRA_MODULES_PATH=/workspace/opencv_contrib/modules \ + -D PYTHON3_EXECUTABLE=/workspace/miniconda3/envs/comfystream/bin/python3.12 \ + -D PYTHON_INCLUDE_DIR=/workspace/miniconda3/envs/comfystream/include/python3.12 \ + -D PYTHON_LIBRARY=/workspace/miniconda3/envs/comfystream/lib/libpython3.12.so \ + -D HAVE_opencv_python3=ON \ + -D WITH_NVCUVID=OFF \ + -D WITH_NVCUVENC=OFF \ + .. && \ + make -j$(nproc) && \ + make install && \ + ldconfig + +# Configure no environment activation by default +RUN conda config --set auto_activate_base false && \ + conda init bash + +WORKDIR /workspace/comfystream From daf4fc557b92d54fa7c297c0e1f68295b8fd5356 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Sat, 27 Sep 2025 08:58:16 +0300 Subject: [PATCH 11/19] chore: update opencv-cuda for Python 3.12, add FasterLivePortrait (#413) * chore: update opencv-cuda for Python 3.12, add FasterLivePortrait add JoyVASA models, add ComfyUI-FasterLivePortrait, add engine build script fix(constraints): add cuda-python dependency to constraints file docker(opencv-cuda): pin move apt pkg installs from entrypoint to dockerfile.base dockerfile: add opencv-cuda installation, update to build v2.1 --------- Co-authored-by: John | Elite Encoder --- configs/models.yaml | 74 +++++++++++++++++++++++++ configs/nodes.yaml | 6 ++ docker/Dockerfile.base | 48 +++++++++------- docker/entrypoint.sh | 19 +++---- src/comfystream/scripts/constraints.txt | 1 + 5 files changed, 118 insertions(+), 30 deletions(-) diff --git a/configs/models.yaml b/configs/models.yaml index 09149f95..be3fce6b 100644 --- a/configs/models.yaml +++ b/configs/models.yaml @@ -74,6 +74,80 @@ models: path: "text_encoders/CLIPText/model.fp16.safetensors" type: "text_encoder" + # JoyVASA models for ComfyUI-FasterLivePortrait + joyvasa_motion_generator: + name: "JoyVASA Motion Generator" + url: "https://huggingface.co/jdh-algo/JoyVASA/resolve/main/motion_generator/motion_generator_hubert_chinese.pt?download=true" + path: "liveportrait_onnx/joyvasa_models/motion_generator_hubert_chinese.pt" + type: "torch" + + joyvasa_audio_model: + name: "JoyVASA Hubert Chinese" + url: "https://huggingface.co/TencentGameMate/chinese-hubert-base/resolve/main/chinese-hubert-base-fairseq-ckpt.pt?download=true" + path: "liveportrait_onnx/joyvasa_models/chinese-hubert-base-fairseq-ckpt.pt" + type: "torch" + + joyvasa_motion_template: + name: "JoyVASA Motion Template" + url: "https://huggingface.co/jdh-algo/JoyVASA/resolve/main/motion_template/motion_template.pkl?download=true" + path: "liveportrait_onnx/joyvasa_models/motion_template.pkl" + type: "pickle" + + # LivePortrait ONNX models - only necessary to build TRT engines + warping_spade: + name: "WarpingSpadeModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/warping_spade-fix.onnx?download=true" + path: "liveportrait_onnx/warping_spade-fix.onnx" + type: "onnx" + + motion_extractor: + name: "MotionExtractorModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/motion_extractor.onnx?download=true" + path: "liveportrait_onnx/motion_extractor.onnx" + type: "onnx" + + landmark: + name: "LandmarkModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/landmark.onnx?download=true" + path: "liveportrait_onnx/landmark.onnx" + type: "onnx" + + face_analysis_retinaface: + name: "FaceAnalysisModel - RetinaFace" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/retinaface_det_static.onnx?download=true" + path: "liveportrait_onnx/retinaface_det_static.onnx" + type: "onnx" + + face_analysis_2dpose: + name: "FaceAnalysisModel - 2DPose" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/face_2dpose_106_static.onnx?download=true" + path: "liveportrait_onnx/face_2dpose_106_static.onnx" + type: "onnx" + + appearance_feature_extractor: + name: "AppearanceFeatureExtractorModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/appearance_feature_extractor.onnx?download=true" + path: "liveportrait_onnx/appearance_feature_extractor.onnx" + type: "onnx" + + stitching: + name: "StitchingModel" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/stitching.onnx?download=true" + path: "liveportrait_onnx/stitching.onnx" + type: "onnx" + + stitching_eye_retarget: + name: "StitchingModel (Eye Retargeting)" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/stitching_eye.onnx?download=true" + path: "liveportrait_onnx/stitching_eye.onnx" + type: "onnx" + + stitching_lip_retarget: + name: "StitchingModel (Lip Retargeting)" + url: "https://huggingface.co/warmshao/FasterLivePortrait/resolve/main/liveportrait_onnx/stitching_lip.onnx?download=true" + path: "liveportrait_onnx/stitching_lip.onnx" + type: "onnx" + sd-turbo: name: "SD-Turbo" url: "https://huggingface.co/stabilityai/sd-turbo/resolve/main/sd_turbo.safetensors" diff --git a/configs/nodes.yaml b/configs/nodes.yaml index 49d422a5..cf497e07 100644 --- a/configs/nodes.yaml +++ b/configs/nodes.yaml @@ -19,6 +19,12 @@ nodes: branch: "main" type: "tensorrt" + comfyui-fasterliveportrait: + name: "ComfyUI FasterLivePortrait" + url: "https://github.com/pschroedl/ComfyUI-FasterLivePortrait.git" + branch: "main" + type: "tensorrt" + # Ryan's nodes comfyui-ryanontheinside: name: "ComfyUI RyanOnTheInside" diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 7d3facda..00c21455 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -8,31 +8,20 @@ ARG CONDA_VERSION \ PYTHON_VERSION ENV DEBIAN_FRONTEND=noninteractive \ + TensorRT_ROOT=/opt/TensorRT-10.12.0.36 \ CONDA_VERSION="${CONDA_VERSION}" \ PATH="/workspace/miniconda3/bin:${PATH}" \ PYTHON_VERSION="${PYTHON_VERSION}" - + # System dependencies RUN apt update && apt install -yqq --no-install-recommends \ - git \ - wget \ - nano \ - socat \ - libsndfile1 \ - build-essential \ - llvm \ - tk-dev \ - libglvnd-dev \ - cmake \ - swig \ - libprotobuf-dev \ - protobuf-compiler \ - libcairo2-dev \ - libpango1.0-dev \ - libgdk-pixbuf2.0-dev \ - libffi-dev \ - libgirepository1.0-dev \ - pkg-config \ + git wget nano socat \ + libsndfile1 build-essential llvm tk-dev \ + libglvnd-dev cmake swig libprotobuf-dev \ + protobuf-compiler libcairo2-dev libpango1.0-dev libgdk-pixbuf2.0-dev \ + libffi-dev libgirepository1.0-dev pkg-config libgflags-dev \ + libgoogle-glog-dev libjpeg-dev libavcodec-dev libavformat-dev \ + libavutil-dev libswscale-dev \ && rm -rf /var/lib/apt/lists/* #enable opengl support with nvidia gpu @@ -61,6 +50,21 @@ conda run -n comfystream --no-capture-output pip install wheel COPY ./src/comfystream/scripts /workspace/comfystream/src/comfystream/scripts COPY ./configs /workspace/comfystream/configs +# TensorRT SDK +WORKDIR /opt +RUN wget --progress=dot:giga \ + https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.12.0/tars/TensorRT-10.12.0.36.Linux.x86_64-gnu.cuda-12.9.tar.gz \ + && tar -xzf TensorRT-10.12.0.36.Linux.x86_64-gnu.cuda-12.9.tar.gz \ + && rm TensorRT-10.12.0.36.Linux.x86_64-gnu.cuda-12.9.tar.gz + +# Link libraries and update linker cache +RUN echo "${TensorRT_ROOT}/lib" > /etc/ld.so.conf.d/tensorrt.conf \ + && ldconfig + +# Install matching TensorRT Python bindings for CPython 3.12 +RUN conda run -n comfystream pip install --no-cache-dir \ + ${TensorRT_ROOT}/python/tensorrt-10.12.0.36-cp312-none-linux_x86_64.whl + # Clone ComfyUI RUN git clone --branch v0.3.56 --depth 1 https://github.com/comfyanonymous/ComfyUI.git /workspace/ComfyUI @@ -72,6 +76,10 @@ RUN conda run -n comfystream --cwd /workspace/comfystream --no-capture-output pi # Copy comfystream and example workflows to ComfyUI COPY ./workflows/comfyui/* /workspace/ComfyUI/user/default/workflows/ COPY ./test/example-512x512.png /workspace/ComfyUI/input +COPY ./docker/entrypoint.sh /workspace/comfystream/docker/entrypoint.sh + +# Install OpenCV CUDA +RUN conda run -n comfystream --no-capture-output --cwd /workspace/comfystream --no-capture-output docker/entrypoint.sh --opencv-cuda # Install ComfyUI requirements RUN conda run -n comfystream --no-capture-output --cwd /workspace/ComfyUI pip install -r requirements.txt --root-user-action=ignore diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index ef55c77e..e6d44463 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -130,6 +130,14 @@ if [ "$1" = "--build-engines" ]; then echo "Engine for DepthAnything2 (large) already exists at ${DEPTH_ANYTHING_DIR}/${DEPTH_ANYTHING_ENGINE_LARGE}, skipping..." fi + # Build Engines for FasterLivePortrait + if [ ! -f "$FASTERLIVEPORTRAIT_DIR/warping_spade-fix.trt" ]; then + cd "$FASTERLIVEPORTRAIT_DIR" + bash /workspace/ComfyUI/custom_nodes/ComfyUI-FasterLivePortrait/scripts/build_fasterliveportrait_trt.sh "${FASTERLIVEPORTRAIT_DIR}" "${FASTERLIVEPORTRAIT_DIR}" "${FASTERLIVEPORTRAIT_DIR}" + else + echo "Engines for FasterLivePortrait already exists, skipping..." + fi + # Build Engine for StreamDiffusion if [ ! -f "$TENSORRT_DIR/StreamDiffusion-engines/stabilityai/sd-turbo--lcm_lora-True--tiny_vae-True--max_batch-3--min_batch-3--mode-img2img/unet.engine.opt.onnx" ]; then cd /workspace/ComfyUI/custom_nodes/ComfyUI-StreamDiffusion @@ -158,7 +166,7 @@ if [ "$1" = "--opencv-cuda" ]; then if [ ! -f "/workspace/comfystream/opencv-cuda-release.tar.gz" ]; then # Download and extract OpenCV CUDA build DOWNLOAD_NAME="opencv-cuda-release.tar.gz" - wget -q -O "$DOWNLOAD_NAME" https://github.com/JJassonn69/ComfyUI-Stream-Pack/releases/download/v2/opencv-cuda-release.tar.gz + wget -q -O "$DOWNLOAD_NAME" https://github.com/JJassonn69/ComfyUI-Stream-Pack/releases/download/v2.1/opencv-cuda-release.tar.gz tar -xzf "$DOWNLOAD_NAME" -C /workspace/comfystream/ rm "$DOWNLOAD_NAME" else @@ -166,15 +174,6 @@ if [ "$1" = "--opencv-cuda" ]; then fi # Install required libraries - apt-get update && apt-get install -y \ - libgflags-dev \ - libgoogle-glog-dev \ - libjpeg-dev \ - libavcodec-dev \ - libavformat-dev \ - libavutil-dev \ - libswscale-dev - # Remove existing cv2 package SITE_PACKAGES_DIR="/workspace/miniconda3/envs/comfystream/lib/python3.12/site-packages" rm -rf "${SITE_PACKAGES_DIR}/cv2"* diff --git a/src/comfystream/scripts/constraints.txt b/src/comfystream/scripts/constraints.txt index 529030f2..9d1bdccb 100644 --- a/src/comfystream/scripts/constraints.txt +++ b/src/comfystream/scripts/constraints.txt @@ -2,6 +2,7 @@ --extra-index-url https://pypi.nvidia.com numpy<2.0.0 torch==2.7.1+cu128 +cuda-python<13.0 torchvision==0.22.1+cu128 torchaudio==2.7.1+cu128 tensorrt==10.12.0.36 From 2909ea4e36aac50856c812fc4376775409ed7a5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Sep 2025 02:09:32 -0400 Subject: [PATCH 12/19] chore(deps): bump actions/setup-node from 4 to 5 (#377) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 4 to 5. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-node dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 01900b71..b46ff8ba 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: node-version: 18 cache: npm From cc292f89f349a17f42b244e128c5a156235f0a21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Sep 2025 02:33:57 -0400 Subject: [PATCH 13/19] chore(deps-dev): bump brace-expansion from 1.1.11 to 1.1.12 in /ui (#443) Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 1.1.11 to 1.1.12. - [Release notes](https://github.com/juliangruber/brace-expansion/releases) - [Commits](https://github.com/juliangruber/brace-expansion/compare/1.1.11...v1.1.12) ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ui/package-lock.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 1261f842..0938cc8b 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -2367,9 +2367,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { @@ -2866,9 +2866,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { @@ -7389,9 +7389,9 @@ } }, "node_modules/sucrase/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" From 50717e0f1970987d85fe16518629cfc8ab273c70 Mon Sep 17 00:00:00 2001 From: Jason <83615043+JJassonn69@users.noreply.github.com> Date: Mon, 29 Sep 2025 17:23:35 +0300 Subject: [PATCH 14/19] Feat/updated UI remove inline (#444) * refactor: remove unused HTTP streaming modules and clean up related code --- server/app.py | 29 ----- server/frame_buffer.py | 42 ------- server/http_streaming/__init__.py | 5 - server/http_streaming/routes.py | 69 ----------- server/http_streaming/tokens.py | 86 ------------- server/public/stream.html | 60 --------- ui/src/app/webrtc-preview/page.tsx | 176 +++++++++++++++++++++++++++ ui/src/components/room.tsx | 6 +- ui/src/components/stream-control.tsx | 127 ++++++++----------- ui/src/hooks/use-peer.ts | 26 +++- 10 files changed, 252 insertions(+), 374 deletions(-) delete mode 100644 server/frame_buffer.py delete mode 100644 server/http_streaming/__init__.py delete mode 100644 server/http_streaming/routes.py delete mode 100644 server/http_streaming/tokens.py delete mode 100644 server/public/stream.html create mode 100644 ui/src/app/webrtc-preview/page.tsx diff --git a/server/app.py b/server/app.py index 028f12bb..b496a458 100644 --- a/server/app.py +++ b/server/app.py @@ -12,9 +12,6 @@ if torch.cuda.is_available(): torch.cuda.init() - -from aiohttp import web, MultipartWriter -from aiohttp_cors import setup as setup_cors, ResourceOptions from aiohttp import web from aiortc import ( MediaStreamTrack, @@ -24,7 +21,6 @@ RTCSessionDescription, ) # Import HTTP streaming modules -from http_streaming.routes import setup_routes from aiortc.codecs import h264 from aiortc.rtcrtpsender import RTCRtpSender from comfystream.pipeline import Pipeline @@ -112,15 +108,6 @@ async def recv(self): """ processed_frame = await self.pipeline.get_processed_video_frame() - # Update the frame buffer with the processed frame - try: - from frame_buffer import FrameBuffer - frame_buffer = FrameBuffer.get_instance() - frame_buffer.update_frame(processed_frame) - except Exception as e: - # Don't let frame buffer errors affect the main pipeline - print(f"Error updating frame buffer: {e}") - # Increment the frame count to calculate FPS. await self.fps_meter.increment_frame_count() @@ -637,16 +624,6 @@ async def on_shutdown(app: web.Application): app = web.Application() app["media_ports"] = args.media_ports.split(",") if args.media_ports else None app["workspace"] = args.workspace - - # Setup CORS - cors = setup_cors(app, defaults={ - "*": ResourceOptions( - allow_credentials=True, - expose_headers="*", - allow_headers="*", - allow_methods=["GET", "POST", "OPTIONS"] - ) - }) app.on_startup.append(on_startup) app.on_shutdown.append(on_shutdown) @@ -658,12 +635,6 @@ async def on_shutdown(app: web.Application): app.router.add_post("/offer", offer) app.router.add_post("/prompt", set_prompt) - # Setup HTTP streaming routes - setup_routes(app, cors) - - # Serve static files from the public directory - app.router.add_static("/", path=os.path.join(os.path.dirname(__file__), "public"), name="static") - # Add routes for getting stream statistics. stream_stats_manager = StreamStatsManager(app) app.router.add_get( diff --git a/server/frame_buffer.py b/server/frame_buffer.py deleted file mode 100644 index 2a16407a..00000000 --- a/server/frame_buffer.py +++ /dev/null @@ -1,42 +0,0 @@ -import threading -import time -import numpy as np -import cv2 -import av -from typing import Optional - -class FrameBuffer: - _instance = None - - @classmethod - def get_instance(cls): - if cls._instance is None: - cls._instance = FrameBuffer() - return cls._instance - - def __init__(self): - self.current_frame = None - self.frame_lock = threading.Lock() - self.last_update_time = 0 - self.quality = 70 # JPEG quality (0-100) - - def update_frame(self, frame): - """Update the current frame in the buffer""" - with self.frame_lock: - # Convert frame to numpy array if it's an av.VideoFrame - if isinstance(frame, av.VideoFrame): - frame_np = frame.to_ndarray(format="rgb24") - else: - frame_np = frame - - # Store the frame as a JPEG-encoded bytes object for efficient serving - _, jpeg_frame = cv2.imencode('.jpg', cv2.cvtColor(frame_np, cv2.COLOR_RGB2BGR), - [cv2.IMWRITE_JPEG_QUALITY, self.quality]) - - self.current_frame = jpeg_frame.tobytes() - self.last_update_time = time.time() - - def get_current_frame(self) -> Optional[bytes]: - """Get the current frame from the buffer""" - with self.frame_lock: - return self.current_frame diff --git a/server/http_streaming/__init__.py b/server/http_streaming/__init__.py deleted file mode 100644 index 4fad17f7..00000000 --- a/server/http_streaming/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -HTTP Streaming module for ComfyStream. - -This module contains components for token management and HTTP streaming routes. -""" diff --git a/server/http_streaming/routes.py b/server/http_streaming/routes.py deleted file mode 100644 index ac309bae..00000000 --- a/server/http_streaming/routes.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -HTTP streaming routes for ComfyStream. - -This module contains the routes for HTTP streaming. -""" -import asyncio -import logging -from aiohttp import web -from frame_buffer import FrameBuffer -from .tokens import cleanup_expired_sessions, validate_token, create_stream_token - -logger = logging.getLogger(__name__) - -async def stream_mjpeg(request): - """Serve an MJPEG stream with token validation""" - # Clean up expired sessions - cleanup_expired_sessions() - - stream_id = request.query.get("token") - - # Validate the stream token - is_valid, error_message = validate_token(stream_id) - if not is_valid: - return web.Response(status=403, text=error_message) - - frame_buffer = FrameBuffer.get_instance() - - # Use a fixed frame delay for 30 FPS - frame_delay = 1.0 / 30 - - response = web.StreamResponse( - status=200, - reason='OK', - headers={ - 'Content-Type': 'multipart/x-mixed-replace; boundary=frame', - 'Cache-Control': 'no-cache', - 'Connection': 'close', - } - ) - await response.prepare(request) - - try: - while True: - jpeg_frame = frame_buffer.get_current_frame() - if jpeg_frame is not None: - await response.write( - b'--frame\r\n' - b'Content-Type: image/jpeg\r\n\r\n' + jpeg_frame + b'\r\n' - ) - await asyncio.sleep(frame_delay) - except (ConnectionResetError, asyncio.CancelledError): - logger.info("MJPEG stream connection closed") - except Exception as e: - logger.error(f"Error in MJPEG stream: {e}") - finally: - return response - -def setup_routes(app, cors): - """Setup HTTP streaming routes - - Args: - app: The aiohttp web application - cors: The CORS setup object - """ - # Stream token endpoints - cors.add(app.router.add_post("/api/stream-token", create_stream_token)) - - # Stream endpoint with token validation - cors.add(app.router.add_get("/api/stream", stream_mjpeg)) diff --git a/server/http_streaming/tokens.py b/server/http_streaming/tokens.py deleted file mode 100644 index d424cf36..00000000 --- a/server/http_streaming/tokens.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Token management system for ComfyStream HTTP streaming. - -This module handles the creation, validation, and management of stream tokens. -""" -import time -import secrets -import logging -from aiohttp import web - -logger = logging.getLogger(__name__) - -# Constants -SESSION_CLEANUP_INTERVAL = 60 # Clean up expired sessions every 60 seconds - -# Global token storage -active_stream_sessions = {} -last_cleanup_time = 0 - -def cleanup_expired_sessions(): - """Clean up expired stream sessions""" - global active_stream_sessions, last_cleanup_time - - current_time = time.time() - - # Only clean up if it's been at least SESSION_CLEANUP_INTERVAL since last cleanup - if current_time - last_cleanup_time < SESSION_CLEANUP_INTERVAL: - return - - # Update the last cleanup time - last_cleanup_time = current_time - - # Find expired sessions - expired_sessions = [sid for sid, expires in active_stream_sessions.items() if current_time > expires] - - # Remove expired sessions - for sid in expired_sessions: - logger.info(f"Removing expired session: {sid[:8]}...") - del active_stream_sessions[sid] - - if expired_sessions: - logger.info(f"Cleaned up {len(expired_sessions)} expired sessions. {len(active_stream_sessions)} active sessions remaining.") - -async def create_stream_token(request): - """Create a unique stream token for secure access to the stream""" - global active_stream_sessions - - # Clean up expired sessions - cleanup_expired_sessions() - - current_time = time.time() - - # Generate a new unique token - stream_id = secrets.token_urlsafe(32) - expires_at = current_time + 3600 # 1 hour from now - - # Store the new session - active_stream_sessions[stream_id] = expires_at - - logger.info(f"Generated new stream token: {stream_id[:8]}... ({len(active_stream_sessions)} active sessions)") - - return web.json_response({ - "stream_id": stream_id, - "expires_at": int(expires_at) - }) - -def validate_token(token): - """Validate a stream token and return whether it's valid - - Args: - token: The token to validate - - Returns: - tuple: (is_valid, error_message) - """ - if not token or token not in active_stream_sessions: - return False, "Invalid stream token" - - # Check if token is expired - current_time = time.time() - if current_time > active_stream_sessions[token]: - # Remove expired token - del active_stream_sessions[token] - return False, "Stream token expired" - - return True, None diff --git a/server/public/stream.html b/server/public/stream.html deleted file mode 100644 index 536781f9..00000000 --- a/server/public/stream.html +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - ComfyStream - OBS Capture - - - - - Video Stream - - diff --git a/ui/src/app/webrtc-preview/page.tsx b/ui/src/app/webrtc-preview/page.tsx new file mode 100644 index 00000000..b2d2045a --- /dev/null +++ b/ui/src/app/webrtc-preview/page.tsx @@ -0,0 +1,176 @@ +"use client"; +// WebRTC Preview Popup Page (client-only) + +import React, { useEffect, useRef, useState, useCallback } from "react"; + +const POLL_INTERVAL_MS = 300; +const MAX_ATTEMPTS = 200; // ~60s + +export default function WebRTCPopupPage() { + const videoRef = useRef(null); + const localStreamRef = useRef(null); + const parentStreamRef = useRef(null); + const clonedIdsRef = useRef>(new Set()); + const attemptsRef = useRef(0); + const intervalRef = useRef(null); + const [status, setStatus] = useState("Initializing…"); + + const clearIntervalInternal = useCallback(() => { + if (intervalRef.current !== null) { + window.clearInterval(intervalRef.current); + intervalRef.current = null; + } + }, []); + + const scheduleClose = useCallback((delay = 800) => { + window.setTimeout(() => { + try { window.close(); } catch { /* noop */ } + }, delay); + }, []); + + const validateOpener = useCallback((): boolean => { + try { + if (!window.opener) { + setStatus("Opener lost. Closing…"); + scheduleClose(); + return false; + } + void window.opener.location.href; // cross-origin check + return true; + } catch { + setStatus("Cross-origin opener. Closing…"); + scheduleClose(); + return false; + } + }, [scheduleClose]); + + const attachVideoIfNeeded = () => { + if (!localStreamRef.current) return; + const video = videoRef.current; + if (video && video.srcObject !== localStreamRef.current) { + video.srcObject = localStreamRef.current; + } + }; + + const cloneTracks = useCallback(() => { + if (!validateOpener()) return; + // @ts-ignore - global from opener context + const parentStream: MediaStream | undefined = window.opener?.__comfystreamRemoteStream; + if (!parentStream) { + setStatus("Waiting for stream…"); + return; + } + if (!localStreamRef.current) { + localStreamRef.current = new MediaStream(); + } + // Parent stream changed -> reset + if (parentStreamRef.current && parentStreamRef.current !== parentStream) { + localStreamRef.current.getTracks().forEach(t => { try { t.stop(); } catch { /* */ } }); + localStreamRef.current = new MediaStream(); + clonedIdsRef.current.clear(); + } + parentStreamRef.current = parentStream; + + let added = false; + parentStream.getTracks().forEach(src => { + if (src.readyState === "ended") return; + if (!clonedIdsRef.current.has(src.id)) { + try { + const clone = src.clone(); + clone.addEventListener("ended", () => { + clonedIdsRef.current.delete(src.id); + }); + localStreamRef.current!.addTrack(clone); + clonedIdsRef.current.add(src.id); + added = true; + } catch { + /* skip */ + } + } + }); + // Cleanup ended clones + localStreamRef.current.getTracks().forEach(t => { + if (t.readyState === "ended") { + localStreamRef.current!.removeTrack(t); + try { t.stop(); } catch { /* */ } + } + }); + if (added) { + attachVideoIfNeeded(); + setStatus("Live"); + videoRef.current?.play().catch(() => {}); + } + }, [validateOpener]); + + useEffect(() => { + if (typeof window === "undefined") return; // safety + attemptsRef.current = 0; + setStatus("Initializing…"); + + const tick = () => { + attemptsRef.current += 1; + if (!validateOpener()) { + clearIntervalInternal(); + return; + } + // @ts-ignore + if (!window.opener.__comfystreamRemoteStream) { + setStatus("Parent stream ended"); + clearIntervalInternal(); + scheduleClose(1200); + return; + } + cloneTracks(); + if (attemptsRef.current >= MAX_ATTEMPTS && (!localStreamRef.current || localStreamRef.current.getTracks().length === 0)) { + setStatus("Timeout waiting for stream"); + clearIntervalInternal(); + scheduleClose(1500); + } + }; + + intervalRef.current = window.setInterval(tick, POLL_INTERVAL_MS); + cloneTracks(); + + const beforeUnload = () => { + clearIntervalInternal(); + localStreamRef.current?.getTracks().forEach(t => { try { t.stop(); } catch { /* */ } }); + }; + window.addEventListener("beforeunload", beforeUnload); + return () => { + window.removeEventListener("beforeunload", beforeUnload); + clearIntervalInternal(); + localStreamRef.current?.getTracks().forEach(t => { try { t.stop(); } catch { /* */ } }); + }; + }, [cloneTracks, clearIntervalInternal, validateOpener, scheduleClose]); + + return ( +
+
+ ); +} diff --git a/ui/src/components/room.tsx b/ui/src/components/room.tsx index 85c6a8f8..df03ed50 100644 --- a/ui/src/components/room.tsx +++ b/ui/src/components/room.tsx @@ -178,12 +178,11 @@ interface StageProps { onStreamReady: () => void; onComfyUIReady: () => void; resolution: { width: number; height: number }; - backendUrl: string; onOutputStreamReady: (stream: MediaStream | null) => void; prompts: Prompt[] | null; } -function Stage({ connected, onStreamReady, onComfyUIReady, resolution, backendUrl, onOutputStreamReady, prompts }: StageProps) { +function Stage({ connected, onStreamReady, onComfyUIReady, resolution, onOutputStreamReady, prompts }: StageProps) { const { remoteStream, peerConnection } = usePeerContext(); const [frameRate, setFrameRate] = useState(0); // Add state and refs for tracking frames @@ -310,7 +309,7 @@ function Stage({ connected, onStreamReady, onComfyUIReady, resolution, backendUr )} {/* Add StreamControlIcon at the bottom right corner of the video box */} - + ); } @@ -598,7 +597,6 @@ export const Room = () => { => { - try { - // Validate backendUrl - if (!backendUrl) { - console.error("Backend URL is not configured."); - throw new Error("Backend URL is not configured in settings."); - } - // Parse base URL from the provided backendUrl - let baseUrl: string; + // Open popup which polls opener for stream and clones tracks locally (no postMessage MediaStream cloning) + const openWebRTCPopup = useCallback(() => { + const features = 'width=1024,height=1024'; + const getBasePath = (): string => { try { - // The origin property gives us "http://hostname:port" - baseUrl = new URL(backendUrl).origin; - } catch (e) { - console.error("Invalid backend URL configured:", backendUrl, e); - throw new Error(`Invalid backend URL configured: ${backendUrl}`); - } - - // Check if we're in a hosted environment by looking at the current URL - // This might need adjustment depending on how hosted environments are detected - const isHosted = window.location.pathname.includes('/live'); - const pathPrefix = isHosted ? '/live' : ''; - - // Request a unique streamID from the server using the derived baseUrl - const response = await fetch(`${baseUrl}${pathPrefix}/api/stream-token`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' + const scripts = document.querySelectorAll('script[src]'); + for (const s of Array.from(scripts)) { + const src = (s as HTMLScriptElement).src; + // Look for /_next/static/ which precedes hashed chunks + const idx = src.indexOf('/_next/static/'); + if (idx !== -1) { + const urlObj = new URL(src); + const before = urlObj.pathname.substring(0, urlObj.pathname.indexOf('/_next/static/')); + if (before !== undefined) { + return before.replace(/\/$/, ''); + } + } } - }); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.error || `Failed to get stream token: ${response.status}`); - } - - const data = await response.json(); - const streamId = data.stream_id; - - // Return the URL with the unique streamID, using the derived baseUrl - // Note: Token will be removed from URL in a later step - return `${baseUrl}${pathPrefix}/stream.html?token=${streamId}`; - } catch (error) { - console.error('Error getting stream URL:', error); - return null; - } - }; - - // Open the stream in a new window - const openStreamWindow = async () => { - try { - setIsLoading(true); - const streamUrl = await getStreamUrl(); - - if (!streamUrl) { - throw new Error('Failed to get stream URL'); - } - - const newWindow = window.open(streamUrl, 'ComfyStream OBS Capture', 'width=1024,height=1024'); - - if (!newWindow) { - throw new Error('Failed to open stream window. Please check your popup blocker settings.'); - } - } catch (error) { - console.error('Error opening stream window:', error); - alert(error instanceof Error ? error.message : 'Failed to open stream window. Please try again.'); - } finally { - setIsLoading(false); + } catch { /* ignore */ } + + try { + const { pathname } = window.location; + // If pathname points to a file (no trailing slash and contains a dot), strip file portion + if (/\.[a-zA-Z0-9]{2,8}$/.test(pathname.split('/').pop() || '')) { + const parts = pathname.split('/'); + parts.pop(); + return parts.join('/') || '/'; + } + return pathname.replace(/\/$/, ''); + } catch { /* ignore */ } + + return ''; + }; + + const basePath = getBasePath(); + const isDev = process.env.NEXT_PUBLIC_DEV === 'true'; + const previewPath = (basePath ? basePath : '') + (isDev ? '/webrtc-preview' : '/webrtc-preview.html'); + + const popup = window.open(previewPath, 'comfystream_preview', features) || window.open(previewPath); + if (!popup) { + alert('Popup blocked. Please allow popups for this site.'); } + }, []); + + const openStreamWindow = () => { + openWebRTCPopup(); }; return (