feat: Implement research time tracking and company detection in secto…

app/sectors/api_routes.py

@@ -571,3 +571,54 @@ def get_sector_sections():
         'icon': s.icon,
         'description': s.description
     } for s in sections])
+
+
+# ============================================================================
+# RESEARCH TIME TRACKING
+# ============================================================================
+
+# Upper bound on a single heartbeat increment. The client flushes roughly every
+# 30s, so anything much larger than a couple of minutes indicates a stale or
+# tampered payload and is rejected rather than trusted.
+MAX_TRACK_TIME_SECONDS = 300
+
+
+@sectors_bp.route('/api/sectors/analysis/<int:analysis_id>/track-time', methods=['POST'])
+@login_required
+def track_research_time(analysis_id):
+    """Increment active research time for a sector analysis.
+
+    Called periodically by the client-side heartbeat while the sector research
+    page is open, visible, and the user is active. Each request adds a small
+    number of seconds to ``total_time_spent``.
+    """
+    analysis = SectorAnalysis.query.filter_by(
+        id=analysis_id, user_id=current_user.id
+    ).first()
+
+    if not analysis:
+        return json_unauthorized('Access denied')
+
+    data = request.get_json(silent=True) or {}
+
+    try:
+        seconds = int(data.get('seconds', 0))
+    except (TypeError, ValueError):
+        return json_error('Invalid seconds value')
+
+    if seconds <= 0 or seconds > MAX_TRACK_TIME_SECONDS:
+        return json_error('Invalid seconds value')
+
+    analysis.total_time_spent = (analysis.total_time_spent or 0) + seconds
+
+    try:
+        db.session.commit()
+    except Exception as e:
+        db.session.rollback()
+        return json_error(str(e), status_code=500)
+
+    return jsonify({
+        'success': True,
+        'total_time_spent': analysis.total_time_spent,
+        'time_spent_formatted': analysis.time_spent_formatted,
+    })

app/sectors/routes.py

@@ -1268,6 +1268,66 @@ def detect_companies():
     })
 
 
+@sectors_bp.route('/<string:sector_name>/scan-companies', methods=['POST'])
+@login_required
+def scan_document_companies(sector_name):
+    """Scan this sector's research text for mentions of tracked companies.
+
+    Unlike the per-note detection that fires on canvas note save, this scans the
+    full research corpus for the sector — the document view content plus every
+    canvas note — and reports which tracked companies are mentioned, along with
+    whether each is already in the sector, in the portfolio, or on the watchlist.
+    The client uses this to let the user pull mentioned holdings into the sector
+    so watchlist/portfolio metrics capture them.
+    """
+    from app.utils.company_detection import detect_company_mentions
+
+    sector = get_sector_by_name_or_slug(sector_name, current_user.id)
+    if not sector:
+        return json_not_found('Sector')
+
+    analysis = SectorAnalysis.query.filter_by(
+        user_id=current_user.id, sector_id=sector.id
+    ).first()
+    if not analysis:
+        return json_not_found('Sector research')
+
+    # Gather the full research corpus: document view + all canvas notes.
+    text_parts = []
+    if analysis.document_content:
+        text_parts.append(analysis.document_content)
+    for note in analysis.canvas_notes.all():
+        if note.title:
+            text_parts.append(note.title)
+        if note.content:
+            text_parts.append(note.content)
+    combined_text = '\n'.join(text_parts)
+
+    matches = detect_company_mentions(combined_text, current_user.id)
+
+    favorite_ids = {c.id for c in current_user.favorites.all()}
+
+    companies = []
+    for match in matches:
+        company = match['company']
+        companies.append({
+            'id': company.id,
+            'name': company.name,
+            'ticker': company.ticker_symbol or '',
+            'matched_text': match['matched_text'],
+            'confidence': match['confidence'],
+            'in_sector': company.sector_id == sector.id,
+            'is_in_portfolio': company.is_in_portfolio,
+            'is_in_watchlist': company.id in favorite_ids,
+        })
+
+    return jsonify({
+        'success': True,
+        'companies': companies,
+        'total': len(companies),
+    })
+
+
 @sectors_bp.route('/note/<int:note_id>/link-companies', methods=['POST'])
 @login_required
 def link_companies_to_note(note_id):

app/sectors/templates/components/archive_modal.html

@@ -34,10 +34,12 @@ <h5 class="modal-title"><i class="bi bi-archive"></i> Archive Sector Research?</
                                 <i class="bi bi-link-45deg"></i>
                                 <span><strong>{{ analysis.sources_count }}</strong> sources added</span>
                             </li>
+                            {# Research progress % hidden with the progress bar (issue #304)
                             <li class="research-summary-item">
                                 <i class="bi bi-graph-up"></i>
                                 <span><strong>{{ analysis.research_progress_score }}%</strong> research progress</span>
                             </li>
+                            #}
                         </ul>
                     </div>
 

app/sectors/templates/sector_analysis.html

@@ -101,6 +101,11 @@ <h2>📊 {{ analysis.sector_name }}</h2>
             </div>
         </div>
 
+        {# Research Progress bar hidden (issue #304): a 0-100% "completeness" score
+           is misleading — research is never "done". Scoring code (research_progress_score,
+           progress_stage, progress_breakdown) is kept dormant pending a future
+           capital-at-risk signal that replaces it. To revive, un-comment this block
+           and the progressBreakdownModal include below.
         <div class="sector-progress-section">
             <div class="sector-progress-row">
                 <span class="sector-progress-label">Research Progress:</span>
@@ -118,6 +123,7 @@ <h2>📊 {{ analysis.sector_name }}</h2>
                 </span>
             </div>
         </div>
+        #}
 
         <div class="sector-metrics-compact">
             <div class="sector-metric-compact">
@@ -494,6 +500,12 @@ <h2 class="accordion-header">
                                 <button class="btn btn-outline-primary btn-sm w-100 mt-2" id="newCompanyBtn">
                                     <i class="bi bi-plus-circle"></i> New Company
                                 </button>
+
+                                <button class="btn btn-outline-secondary btn-sm w-100 mt-2" id="scanCompaniesBtn"
+                                        title="Find companies you already track that are mentioned in your research text">
+                                    <i class="bi bi-search"></i> Detect companies in research
+                                </button>
+                                <div id="scanCompaniesResults" class="mt-2"></div>
                             </div>
                         </div>
                     </div>
@@ -566,7 +578,8 @@ <h2 class="accordion-header">
 
 {% block modals %}
 <!-- Sector-specific modals -->
-{% include 'components/progress_breakdown_modal.html' %}
+{# Progress breakdown ("How Scoring Works") modal hidden with the progress bar (issue #304) #}
+{# {% include 'components/progress_breakdown_modal.html' %} #}
 {% include 'components/templates_modal.html' %}
 <!-- Tips Modal -->
 {% set tips_title = "Sector Research Tips" %}
@@ -593,6 +606,11 @@ <h2 class="accordion-header">
 {% block scripts %}
 {{ super() }}
 {% include 'components/_tabulator_js.html' %}
+
+<!-- Active research time tracker (heartbeat) -->
+<span id="sector-time-tracker" data-analysis-id="{{ analysis.id }}" hidden></span>
+<script src="{{ url_for('static', filename='js/sector-time-tracker.js') }}" defer></script>
+
 <script>
 // Continuous Learning Toggle Handler
 document.getElementById('continuousLearningToggle').addEventListener('change', function() {

app/sectors/templates/sector_analysis_focus.html

@@ -447,6 +447,10 @@ <h3>
     }
 });
 </script>
+
+<!-- Active research time tracker (heartbeat) -->
+<span id="sector-time-tracker" data-analysis-id="{{ analysis.id }}" hidden></span>
+<script src="{{ url_for('static', filename='js/sector-time-tracker.js') }}" defer></script>
 {% endblock %}
 
 {% block modals %}

app/static/js/sector-time-tracker.js

@@ -0,0 +1,84 @@
+/**
+ * Sector research time tracker.
+ *
+ * Accumulates *active* research time on the sector analysis page and flushes it
+ * to the server periodically. "Active" means the tab is visible AND the user has
+ * interacted within the idle window — so leaving the tab open in the background
+ * or walking away does not inflate the recorded research time.
+ *
+ * Requires an element with id="sector-time-tracker" carrying a
+ * data-analysis-id attribute, rendered by the sector analysis template.
+ */
+(function () {
+  'use strict';
+
+  var el = document.getElementById('sector-time-tracker');
+  if (!el) return;
+
+  var analysisId = el.getAttribute('data-analysis-id');
+  if (!analysisId) return;
+
+  var endpoint = '/api/sectors/analysis/' + analysisId + '/track-time';
+
+  var IDLE_LIMIT_MS = 60 * 1000; // treat user as idle after 60s of no interaction
+  var FLUSH_INTERVAL_MS = 30 * 1000; // push accumulated time to server every 30s
+  var TICK_MS = 1000;
+
+  var lastActivity = Date.now();
+  var lastTick = Date.now();
+  var pendingSeconds = 0; // active seconds accumulated but not yet flushed
+
+  var activityEvents = ['mousemove', 'keydown', 'scroll', 'click', 'touchstart'];
+  activityEvents.forEach(function (evt) {
+    document.addEventListener(evt, function () {
+      lastActivity = Date.now();
+    }, { passive: true });
+  });
+
+  // Accumulate elapsed wall-clock time, but only count it while the page is
+  // visible and the user has been active recently.
+  setInterval(function () {
+    var now = Date.now();
+    var deltaSeconds = (now - lastTick) / 1000;
+    lastTick = now;
+
+    var isVisible = document.visibilityState === 'visible';
+    var isActive = (now - lastActivity) < IDLE_LIMIT_MS;
+
+    if (isVisible && isActive) {
+      pendingSeconds += deltaSeconds;
+    }
+  }, TICK_MS);
+
+  function flush(useBeacon) {
+    var seconds = Math.round(pendingSeconds);
+    if (seconds <= 0) return;
+
+    pendingSeconds -= seconds;
+    var payload = JSON.stringify({ seconds: seconds });
+
+    if (useBeacon && navigator.sendBeacon) {
+      var blob = new Blob([payload], { type: 'application/json' });
+      var ok = navigator.sendBeacon(endpoint, blob);
+      if (!ok) pendingSeconds += seconds; // re-queue if the beacon was rejected
+      return;
+    }
+
+    fetch(endpoint, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' },
+      body: payload,
+      keepalive: true,
+    }).catch(function () {
+      pendingSeconds += seconds; // re-queue on network failure
+    });
+  }
+
+  setInterval(function () { flush(false); }, FLUSH_INTERVAL_MS);
+
+  document.addEventListener('visibilitychange', function () {
+    if (document.visibilityState === 'hidden') flush(true);
+  });
+
+  window.addEventListener('beforeunload', function () { flush(true); });
+})();

app/utils/company_detection.py

@@ -65,18 +65,21 @@ def detect_company_mentions(text: str, user_id: int) -> List[Dict]:
             continue
 
         # Priority 1: Exact ticker match (highest confidence)
-        ticker = company.ticker_symbol.upper()
-        # Match ticker as standalone word (not part of another word)
-        ticker_pattern = r'\b' + re.escape(ticker) + r'\b'
-        if re.search(ticker_pattern, text_upper):
-            matches.append({
-                'company': company,
-                'confidence': 'high',
-                'matched_text': company.ticker_symbol,
-                'match_type': 'ticker'
-            })
-            seen_company_ids.add(company.id)
-            continue
+        # Companies may not have a ticker (e.g. private/manually added), so guard
+        # against None before matching.
+        ticker = (company.ticker_symbol or '').strip().upper()
+        if ticker:
+            # Match ticker as standalone word (not part of another word)
+            ticker_pattern = r'\b' + re.escape(ticker) + r'\b'
+            if re.search(ticker_pattern, text_upper):
+                matches.append({
+                    'company': company,
+                    'confidence': 'high',
+                    'matched_text': company.ticker_symbol,
+                    'match_type': 'ticker'
+                })
+                seen_company_ids.add(company.id)
+                continue
 
         # Priority 2: Exact company name match (high confidence)
         company_name = company.name