From ef2275f384bee2b0a9ee006d25f701903092d780 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:17:49 +1100 Subject: [PATCH] media: apple: isp: report the frame gain and exposure Userspace has no way to tell how hard the sensor is working. Gain is the signal a camera pipeline needs to decide when the picture is noisy enough to denoise, and the firmware already writes it into every frame's metadata buffer, which the driver maps but never reads. The metadata buffer opens with a table of section offsets. On H17 firmware the fourth section starts with the frame's exposure: the integration time in microseconds, then the total sensor gain in 1/256 steps with 256 as unity. This was established on T6040 and T8140 by watching auto exposure converge on a fixed scene, where the gain climbed from 260 to its final value once the integration time hit the frame period, and by stepping the frame rate, which capped the integration time at the frame period. Read the section when a metadata buffer comes back, through the table and bounded by the buffer size, and report the last values as read-only, volatile V4L2_CID_GAIN and V4L2_CID_EXPOSURE_ABSOLUTE controls. Only hardware whose match data says its metadata carries the report gets the controls. (cherry picked from commit 3c5441114af9f9e6f93fadf94710269b27676b54) Port to integration/aurora: only T8140 exists on this base; omit the T6040/P010 dependencies. Keep T8122 metadata reporting disabled because the 26.6.2 layout and units are unqualified. Bound the table-entry read as well as its target and test short headers and control admission. Signed-off-by: Ryan Murray Signed-off-by: Satya Benson --- drivers/media/platform/apple/isp/isp-drv.c | 2 + drivers/media/platform/apple/isp/isp-drv.h | 9 ++ .../media/platform/apple/isp/isp-v4l2-test.c | 78 +++++++++++++++ drivers/media/platform/apple/isp/isp-v4l2.c | 95 ++++++++++++++++++- 4 files changed, 183 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 2cbdcd2fb00bde..9572f820e96416 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -685,6 +685,7 @@ static const struct apple_isp_hw apple_isp_hw_t6020 = { .meta_size = ISP_META_SIZE_T8112, }; +/* T8122/26.6.2 metadata layout and units are not qualified: no gain control. */ static const struct apple_isp_hw apple_isp_hw_t8122 = { .gen = ISP_GEN_T8112, .pmu_base = 0x2d0704000, @@ -799,6 +800,7 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .boot_mode = 1, .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, .resident_fw = true, + .meta_exposure = true, }; static const struct of_device_id apple_isp_of_match[] = { diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 8f945c26f00e6d..3e4b94f744c36c 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -189,6 +189,9 @@ struct apple_isp_hw { /* size of a capture metadata buffer, 0 for no such pool */ u32 capture_meta_size; + /* metadata buffers carry the frame's exposure report (H17 firmware) */ + bool meta_exposure; + /* * The firmware can be started only once per system boot, so it is * started at probe and kept running while the driver is bound. @@ -276,6 +279,12 @@ struct apple_isp { struct v4l2_device v4l2_dev; struct v4l2_ctrl_handler ctrl_handler; struct v4l2_ctrl *exposure_priority; + /* read-only exposure report of the last frame, NULL without one */ + struct v4l2_ctrl *gain; + struct v4l2_ctrl *exposure; + /* last reported total gain (1/256 steps) and exposure (us) */ + u32 frame_gain; + u32 frame_exposure_us; struct vb2_queue vbq; struct mutex video_lock; unsigned int sequence; diff --git a/drivers/media/platform/apple/isp/isp-v4l2-test.c b/drivers/media/platform/apple/isp/isp-v4l2-test.c index 895a6d1299c220..b954dac3357368 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2-test.c +++ b/drivers/media/platform/apple/isp/isp-v4l2-test.c @@ -107,7 +107,85 @@ static void isp_test_create_bufs_limit(struct kunit *test) sizes), -EINVAL); } +/* The exposure report is found through the section table, never past it. */ +static void isp_test_meta_exposure(struct kunit *test) +{ + static const struct apple_isp_hw hw = { .meta_exposure = true }; + static const struct apple_isp_hw hw_none = { }; + struct apple_isp *isp = isp_test_alloc(test); + struct isp_surf meta = { .size = 0x2000 }; + u8 *virt = kunit_kzalloc(test, meta.size, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, virt); + meta.virt = virt; + isp->hw = &hw; + isp->frame_gain = 256; + + /* fourth table entry: the section at 0x1870 */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le32(32475, virt + 0x1870); + put_unaligned_le16(0x0800, virt + 0x1874); + put_unaligned_le16(3253, virt + 0x1876); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* A short header cannot even supply the section-table entry. */ + put_unaligned_le16(700, virt + 0x1876); + meta.size = 0x1f; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + meta.size = 0x2000; + put_unaligned_le16(3253, virt + 0x1876); + + /* a section that would end past the buffer is ignored */ + put_unaligned_le32(meta.size - 4, virt + 0x1c); + put_unaligned_le32(1000, virt + meta.size - 4); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* so is an unwritten section */ + put_unaligned_le32(0x1000, virt + 0x1c); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + + /* and firmware without the report */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le16(600, virt + 0x1876); + isp->hw = &hw_none; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + isp->hw = &hw; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 600); +} + +static void isp_test_exposure_control_gate(struct kunit *test) +{ + static const struct apple_isp_hw hw_none = { }; + static const struct apple_isp_hw hw = { .meta_exposure = true }; + struct apple_isp *isp = isp_test_alloc(test); + + v4l2_ctrl_handler_init(&isp->ctrl_handler, 3); + isp->hw = &hw_none; + isp_init_exposure_ctrls(isp); + KUNIT_EXPECT_PTR_EQ(test, isp->gain, NULL); + KUNIT_EXPECT_PTR_EQ(test, isp->exposure, NULL); + isp->hw = &hw; + isp_init_exposure_ctrls(isp); + KUNIT_EXPECT_EQ(test, isp->ctrl_handler.error, 0); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, isp->gain); + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, isp->exposure); + if (isp->gain) + KUNIT_EXPECT_EQ(test, isp->gain->flags & + (V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_VOLATILE), + (u32)(V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_VOLATILE)); + v4l2_ctrl_handler_free(&isp->ctrl_handler); +} + static struct kunit_case isp_v4l2_test_cases[] = { + KUNIT_CASE(isp_test_meta_exposure), + KUNIT_CASE(isp_test_exposure_control_gate), KUNIT_CASE(isp_test_queue_init), KUNIT_CASE(isp_test_reqbufs_limit), KUNIT_CASE(isp_test_create_bufs_limit), diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 072b8523279c28..41ae504b82dc7f 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -57,6 +58,46 @@ static_assert(ISP_CMD_AREA_SIZE(0) >= ISP_IPC_BUFEXC_STAT_SIZE); */ #define ISP_BUFLIST_POOL_CAPTURE_META 2 +/* + * A metadata buffer starts with a table of section offsets. On H17 firmware + * the fourth section opens with the frame's exposure: the integration time in + * microseconds (le32 at 0) and the total sensor gain in 1/256 steps, 256 for + * unity (le16 at 6). Observed on T6040 and T8140 with AE converging under a + * fixed scene and with the frame rate capping the integration time. + */ +#define ISP_META_SECTIONS 0x10 +#define ISP_META_SECTION_EXPOSURE 3 +#define ISP_META_EXPOSURE_TIME 0x0 +#define ISP_META_EXPOSURE_GAIN 0x6 +#define ISP_META_EXPOSURE_SIZE 0x8 + +static void isp_meta_read_exposure(struct apple_isp *isp, + const struct isp_surf *meta) +{ + const u8 *virt = meta->virt; + u32 offset, time; + u16 gain; + + if (!isp->hw->meta_exposure || !virt || + meta->size < ISP_META_SECTIONS + 4 * (ISP_META_SECTION_EXPOSURE + 1)) + return; + + offset = get_unaligned_le32(virt + ISP_META_SECTIONS + + 4 * ISP_META_SECTION_EXPOSURE); + if (offset < ISP_META_SECTIONS || offset > meta->size || + meta->size - offset < ISP_META_EXPOSURE_SIZE) + return; + + time = get_unaligned_le32(virt + offset + ISP_META_EXPOSURE_TIME); + gain = get_unaligned_le16(virt + offset + ISP_META_EXPOSURE_GAIN); + /* An unwritten section or one this layout does not describe */ + if (!time || !gain) + return; + + WRITE_ONCE(isp->frame_exposure_us, time); + WRITE_ONCE(isp->frame_gain, gain); +} + int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) { struct isp_message *req = &chan->req, *rsp = &chan->rsp; @@ -100,6 +141,7 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) struct isp_surf *meta = isp->meta_surfs[j]; if ((u32)bufd->iovas[0] == (u32)meta->iova) { WARN_ON(!meta->submitted); + isp_meta_read_exposure(isp, meta); meta->submitted = false; } } @@ -1037,6 +1079,56 @@ static int isp_vb2_queue_init(struct apple_isp *isp) return vb2_queue_init(vbq); } +static int isp_g_volatile_ctrl(struct v4l2_ctrl *ctrl) +{ + struct apple_isp *isp = + container_of(ctrl->handler, struct apple_isp, ctrl_handler); + + switch (ctrl->id) { + case V4L2_CID_GAIN: + ctrl->val = READ_ONCE(isp->frame_gain); + return 0; + case V4L2_CID_EXPOSURE_ABSOLUTE: + /* in 100 us units */ + ctrl->val = DIV_ROUND_CLOSEST(READ_ONCE(isp->frame_exposure_us), + 100); + return 0; + } + + return -EINVAL; +} + +static const struct v4l2_ctrl_ops isp_exposure_ctrl_ops = { + .g_volatile_ctrl = isp_g_volatile_ctrl, +}; + +/* + * Auto exposure picks the gain and exposure of each frame. Report the last + * frame's values read-only, so that userspace can follow the scene light, for + * example to decide when to denoise. The gain is the firmware's total sensor + * gain in 1/256 steps. + */ +static void isp_init_exposure_ctrls(struct apple_isp *isp) +{ + if (!isp->hw->meta_exposure) + return; + + isp->frame_gain = 256; + isp->gain = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, V4L2_CID_GAIN, + 0, U16_MAX, 1, 256); + isp->exposure = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, + V4L2_CID_EXPOSURE_ABSOLUTE, + 0, 10000, 1, 0); + if (isp->gain) + isp->gain->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; + if (isp->exposure) + isp->exposure->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; +} + int apple_isp_setup_video(struct apple_isp *isp) { struct video_device *vdev = &isp->vdev; @@ -1092,10 +1184,11 @@ int apple_isp_setup_video(struct apple_isp *isp) * Auto exposure is always on. By default it holds the frame rate, as * V4L2 specifies; with exposure priority it may slow down in low light. */ - v4l2_ctrl_handler_init(&isp->ctrl_handler, 1); + v4l2_ctrl_handler_init(&isp->ctrl_handler, 3); isp->exposure_priority = v4l2_ctrl_new_std(&isp->ctrl_handler, NULL, V4L2_CID_EXPOSURE_AUTO_PRIORITY, 0, 1, 1, 0); + isp_init_exposure_ctrls(isp); if (isp->ctrl_handler.error) { err = isp->ctrl_handler.error; dev_err(isp->dev, "failed to create controls: %d\n", err);