diff --git a/.github/workflows/flutter.yml b/.github/workflows/flutter.yml new file mode 100644 index 0000000000..edaed3d198 --- /dev/null +++ b/.github/workflows/flutter.yml @@ -0,0 +1,43 @@ +# This workflow builds and checks the Flutter controller app +# For more information see: https://docs.github.com/actions/guides/building-and-testing-flutter-applications + +name: Flutter CI for Controller App + +on: + push: + branches: [ master ] + paths: + - 'controller/flutter/**' + - '.github/workflows/flutter.yml' + pull_request: + branches: [ master ] + paths: + - 'controller/flutter/**' + - '.github/workflows/flutter.yml' + +jobs: + build: + name: Flutter Checks + runs-on: ubuntu-latest + defaults: + run: + working-directory: controller/flutter + steps: + - uses: actions/checkout@v4 + - name: Set up Flutter + uses: subosito/flutter-action@v2 + with: + flutter-version: '3.35.1' + channel: 'stable' + cache: true + - name: Install dependencies + run: flutter pub get + - name: Analyze + run: flutter analyze --no-fatal-infos + - name: Build APK + run: flutter build apk --debug + - name: Upload Controller App (Flutter) + uses: actions/upload-artifact@v4 + with: + name: controller-flutter-debug.apk + path: controller/flutter/build/app/outputs/flutter-apk/app-debug.apk diff --git a/android/robot/src/main/java/org/openbot/app/robot/env/PhoneController.java b/android/robot/src/main/java/org/openbot/app/robot/env/PhoneController.java index 1f6150598a..e8d6df72e2 100644 --- a/android/robot/src/main/java/org/openbot/app/robot/env/PhoneController.java +++ b/android/robot/src/main/java/org/openbot/app/robot/env/PhoneController.java @@ -108,7 +108,9 @@ private void addVideoView(View videoView, Context context) { } public void connect(Context context) { - setWebVideoEnabled(false); + // Phone mode streams video over the same local connection as web mode does + // over the signaling websocket, so it needs the video server enabled too. + setWebVideoEnabled(true); ILocalConnection connection = connectionSelector.getConnection(); if (!connection.isConnected()) { @@ -250,4 +252,4 @@ private PhoneController() { "Use getInstance() method to get the single instance of this class."); } } -} +} \ No newline at end of file diff --git a/android/robot/src/main/java/org/openbot/app/robot/env/WebRtcServer.java b/android/robot/src/main/java/org/openbot/app/robot/env/WebRtcServer.java index 2902200b1c..0b2ab7aaa3 100644 --- a/android/robot/src/main/java/org/openbot/app/robot/env/WebRtcServer.java +++ b/android/robot/src/main/java/org/openbot/app/robot/env/WebRtcServer.java @@ -12,6 +12,8 @@ import androidx.core.content.ContextCompat; import com.pedro.rtplibrary.view.OpenGlView; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.json.JSONException; import org.json.JSONObject; import org.openbot.app.robot.utils.AndGate; @@ -34,6 +36,7 @@ import org.webrtc.PeerConnection; import org.webrtc.PeerConnectionFactory; import org.webrtc.RtpReceiver; +import org.webrtc.RtpSender; import org.webrtc.RtpTransceiver; import org.webrtc.SessionDescription; import org.webrtc.SurfaceTextureHelper; @@ -81,6 +84,8 @@ public class WebRtcServer implements IVideoServer { SurfaceTextureHelper surfaceTextureHelper; private PeerConnection peerConnection; MediaStream mediaStream; + private RtpSender videoSender; + private RtpSender audioSender; private AndGate andGate; private Context context; @@ -242,7 +247,7 @@ public void onCameraSwitchError(String errorDescription) { private void resumeLocalCamera() { emitLocalCameraCommand(Constants.CMD_RESUME_LOCAL_CAMERA); } - + private void emitLocalCameraCommand(String command) { try { ControllerToBotEventBus.emitEvent(new JSONObject().put("command", command).toString()); @@ -274,20 +279,56 @@ public void onCreateSuccess(SessionDescription sessionDescription) { new MediaConstraints()); } + // Logs whether the negotiated SDP actually contains a video m-line, so a + // logcat pull tells us immediately whether the break is in track + // negotiation (this log) vs. transport/rendering (later logs). + private void logSdp(String label, SessionDescription sdp) { + boolean hasVideo = sdp.description.contains("m=video"); + boolean hasAudio = sdp.description.contains("m=audio"); + Log.d(TAG, "SDP [" + label + "]: m=video present=" + hasVideo + " m=audio present=" + hasAudio); + } + private void startStreamingVideo() { mediaStream = factory.createLocalMediaStream("ARDAMS"); mediaStream.addTrack(videoTrackFromCamera); mediaStream.addTrack(localAudioTrack); - peerConnection.addStream(mediaStream); + // peerConnection.addStream() is a Plan-B-only API and is a no-op under the + // Unified Plan SDP semantics that the current WebRTC build defaults to, so + // tracks must be attached with addTrack() instead. + List streamIds = Collections.singletonList(mediaStream.getId()); + videoSender = peerConnection.addTrack(videoTrackFromCamera, streamIds); + audioSender = peerConnection.addTrack(localAudioTrack, streamIds); + Log.d( + TAG, + "startStreamingVideo: videoSender=" + + (videoSender != null) + + " audioSender=" + + (audioSender != null) + + " videoTrack.state=" + + videoTrackFromCamera.state()); } private void stopStreamingVideo() { - peerConnection.removeStream(mediaStream); + if (videoSender != null) { + peerConnection.removeTrack(videoSender); + } + if (audioSender != null) { + peerConnection.removeTrack(audioSender); + } } private void stopServer() { - mediaStream.removeTrack(videoTrackFromCamera); - mediaStream.removeTrack(localAudioTrack); + if (videoCapturer != null) { + try { + videoCapturer.stopCapture(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + if (peerConnection != null) { + peerConnection.close(); + peerConnection = null; + } view.release(); stopClient(); } @@ -335,6 +376,7 @@ private PeerConnection createPeerConnection(PeerConnectionFactory factory) { iceServers.add(stunServer); PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers); + rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; MediaConstraints pcConstraints = new MediaConstraints(); PeerConnection.Observer pcObserver = @@ -433,6 +475,10 @@ private void sendMessage(JSONObject message) { private void createVideoTrackFromCameraAndShowIt() { audioConstraints = new MediaConstraints(); videoCapturer = createVideoCapturer(); + if (videoCapturer == null) { + Log.e(TAG, "createVideoTrackFromCameraAndShowIt: no back-facing camera capturer found"); + return; + } VideoSource videoSource = factory.createVideoSource(videoCapturer.isScreencast()); surfaceTextureHelper = @@ -454,15 +500,18 @@ private void createVideoTrackFromCameraAndShowIt() { } private void initializePeerConnectionFactory() { + // PeerConnectionFactory.initialize() loads the native WebRTC library; it must + // run before any class that calls into native code (e.g. DefaultVideoEncoderFactory's + // SoftwareVideoEncoderFactory) is constructed, or it fails with UnsatisfiedLinkError. + PeerConnectionFactory.InitializationOptions initializationOptions = + PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions(); + PeerConnectionFactory.initialize(initializationOptions); + VideoEncoderFactory encoderFactory = new DefaultVideoEncoderFactory(rootEglBase.getEglBaseContext(), true, true); VideoDecoderFactory decoderFactory = new DefaultVideoDecoderFactory(rootEglBase.getEglBaseContext()); - PeerConnectionFactory.InitializationOptions initializationOptions = - PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions(); - PeerConnectionFactory.initialize(initializationOptions); - PeerConnectionFactory.Options options = new PeerConnectionFactory.Options(); options.networkIgnoreMask = 16; options.disableEncryption = false; @@ -495,12 +544,45 @@ private boolean useCamera2() { return Camera2Enumerator.isSupported(context); } + private final CameraVideoCapturer.CameraEventsHandler cameraEventsHandler = + new CameraVideoCapturer.CameraEventsHandler() { + @Override + public void onCameraError(String errorDescription) { + Log.e(TAG, "Camera onCameraError: " + errorDescription); + } + + @Override + public void onCameraDisconnected() { + Log.e(TAG, "Camera onCameraDisconnected"); + } + + @Override + public void onCameraFreezed(String errorDescription) { + Log.e(TAG, "Camera onCameraFreezed: " + errorDescription); + } + + @Override + public void onCameraOpening(String cameraName) { + Log.d(TAG, "Camera onCameraOpening: " + cameraName); + } + + @Override + public void onFirstFrameAvailable() { + Log.d(TAG, "Camera onFirstFrameAvailable"); + } + + @Override + public void onCameraClosed() { + Log.d(TAG, "Camera onCameraClosed"); + } + }; + private VideoCapturer createCameraCapturer(CameraEnumerator enumerator) { final String[] deviceNames = enumerator.getDeviceNames(); for (String deviceName : deviceNames) { if (enumerator.isBackFacing(deviceName)) { - VideoCapturer videoCapturer = enumerator.createCapturer(deviceName, null); + VideoCapturer videoCapturer = enumerator.createCapturer(deviceName, cameraEventsHandler); if (videoCapturer != null) { return videoCapturer; } diff --git a/controller/flutter/README.md b/controller/flutter/README.md index ff6eb8a56f..4caa3bada3 100644 --- a/controller/flutter/README.md +++ b/controller/flutter/README.md @@ -47,6 +47,34 @@ Begin by installing [Flutter](https://flutter.dev/) on your system. Choose the a

+## Building a Signed Release (Android) +Release builds (`flutter build apk --release` / `flutter build appbundle --release`) are signed using a keystore referenced from `android/key.properties`. This file is **gitignored** and never committed — each developer must create their own local copy. Debug builds (`flutter build apk --debug`, `flutter run`) don't need any of this. + +### 1. Generate a keystore (once) +```bash +keytool -genkey -v -keystore ~/release.keystore -alias upload -keyalg RSA -keysize 2048 -validity 10000 +``` +Keep the resulting `.keystore`/`.jks` file and its passwords somewhere safe outside the repo — losing it means you can no longer publish updates under the same app signature. + +### 2. Create `android/key.properties` +`android/app/build.gradle` reads this file at `controller/flutter/android/key.properties`, so create it there with: +```properties +storePassword= +keyPassword= +keyAlias=upload +storeFile=/absolute/path/to/release.keystore +``` +`storeFile` must be an absolute path (or a path resolvable from `controller/flutter/android/`) to the keystore you generated in step 1. + +### 3. Build +```bash +cd controller/flutter +flutter build appbundle --release # for Play Store upload (.aab) +# or +flutter build apk --release # for a signed installable APK +``` +`app/build.gradle` loads `key.properties` automatically if present and wires it into the `release` signing config; if the file is missing, the release build config falls back to null signing values and the build will fail signing (debug builds are unaffected). + ## Connection When the controller app is started, it immediately tries to connect to the robot and shows the following screen: diff --git a/controller/flutter/android/app/build.gradle b/controller/flutter/android/app/build.gradle index e7b46d70ac..3030d1f60a 100644 --- a/controller/flutter/android/app/build.gradle +++ b/controller/flutter/android/app/build.gradle @@ -4,8 +4,14 @@ plugins { id "dev.flutter.flutter-gradle-plugin" } +def keystoreProperties = new Properties() +def keystorePropertiesFile = rootProject.file("key.properties") +if (keystorePropertiesFile.exists()) { + keystoreProperties.load(new FileInputStream(keystorePropertiesFile)) +} + android { - namespace "org.openbot.flutter_controller" + namespace "org.openbot.app.controller" // 1. Updated to match the new plugin requirements compileSdk 36 @@ -14,7 +20,7 @@ android { ndkVersion = "28.2.13676358" defaultConfig { - applicationId "org.openbot.flutter_controller" + applicationId "org.openbot.app.controller" // 3. Changed from 'flutter.minSdkVersion' to 24 to support modern WebRTC/Sensors minSdkVersion 24 @@ -26,9 +32,18 @@ android { versionName flutter.versionName } + signingConfigs { + release { + keyAlias keystoreProperties['keyAlias'] + keyPassword keystoreProperties['keyPassword'] + storeFile keystoreProperties['storeFile'] ? file(keystoreProperties['storeFile']) : null + storePassword keystoreProperties['storePassword'] + } + } + buildTypes { release { - signingConfig signingConfigs.debug + signingConfig signingConfigs.release } } diff --git a/controller/flutter/android/app/src/debug/AndroidManifest.xml b/controller/flutter/android/app/src/debug/AndroidManifest.xml index eac323660b..1f82b40ee8 100644 --- a/controller/flutter/android/app/src/debug/AndroidManifest.xml +++ b/controller/flutter/android/app/src/debug/AndroidManifest.xml @@ -1,5 +1,5 @@ + package="org.openbot.app.controller">