Merge branch 'master' of github.com:dimagi/commcare-hq into em/write-…

…domain-metrics

corehq/apps/api/resources/v0_1.py

@@ -17,6 +17,7 @@
 from corehq.apps.groups.models import Group
 from corehq.apps.user_importer.helpers import UserChangeLogger
 from corehq.apps.users.models import CommCareUser, HqPermissions, WebUser
+from corehq.apps.users.util import user_location_data
 from corehq.const import USER_CHANGE_VIA_API
 
 
@@ -105,6 +106,14 @@ def dehydrate(self, bundle):
 
     def dehydrate_user_data(self, bundle):
         user_data = bundle.obj.get_user_data(bundle.obj.domain).to_dict()
+        if location_id := bundle.obj.get_location_id(bundle.obj.domain):
+            # This is all available in the top level, but add in here for backwards-compatibility
+            user_data['commcare_location_id'] = location_id
+            user_data['commcare_location_ids'] = user_location_data(
+                bundle.obj.get_location_ids(bundle.obj.domain))
+            user_data['commcare_primary_case_sharing_id'] = location_id
+
+        user_data['commcare_project'] = bundle.obj.domain
         if self.determine_format(bundle.request) == 'application/xml':
             # attribute names can't start with digits in xml
             user_data = {k: v for k, v in user_data.items() if not k[0].isdigit()}

corehq/apps/builds/tests/test_utils.py

@@ -0,0 +1,30 @@
+from django.test import SimpleTestCase
+from corehq.apps.builds.utils import is_out_of_date
+
+
+class TestVersionUtils(SimpleTestCase):
+
+    def test_is_out_of_date(self):
+        test_cases = [
+            # (version_in_use, latest_version, expected_result)
+            ('2.53.0', '2.53.1', True),     # Normal case - out of date
+            ('2.53.1', '2.53.1', False),    # Same version - not out of date
+            ('2.53.2', '2.53.1', False),    # Higher version - not out of date
+            (None, '2.53.1', False),        # None version_in_use
+            ('2.53.1', None, False),        # None latest_version
+            ('invalid', '2.53.1', False),   # Invalid version string
+            ('2.53.1', 'invalid', False),   # Invalid latest version
+            ('6', '7', True),               # Normal case - app version is integer
+            (None, None, False),            # None version_in_use and latest_version
+            ('2.54', '2.54.0', False),      # Edge case - should not be out of date
+            ('2.54.0', '2.54', False),       # Edge case - should not be out of date
+        ]
+
+        for version_in_use, latest_version, expected in test_cases:
+            with self.subTest(version_in_use=version_in_use, latest_version=latest_version):
+                result = is_out_of_date(version_in_use, latest_version)
+                self.assertEqual(
+                    result,
+                    expected,
+                    f"Expected is_out_of_date('{version_in_use}', '{latest_version}') to be {expected}"
+                )

corehq/apps/builds/utils.py

@@ -1,6 +1,6 @@
 import re
 
-from .models import CommCareBuild, CommCareBuildConfig
+from corehq.apps.builds.models import CommCareBuild, CommCareBuildConfig
 
 
 def get_all_versions(versions):
@@ -40,3 +40,71 @@ def extract_build_info_from_filename(content_disposition):
     else:
         raise ValueError('Could not find filename like {!r} in {!r}'.format(
             pattern, content_disposition))
+
+
+def get_latest_version_at_time(config, target_time, cache=None):
+    """
+    Get the latest CommCare version that was available at a given time.
+    Excludes superuser-only versions.
+    Menu items are already in chronological order (newest last).
+    If no target time is provided, return the latest version available now.
+
+    Args:
+        config: CommCareBuildConfig instance
+        target_time: datetime, or None for latest version
+        cache: Dictionary for per-request caching
+    """
+    if cache is None:
+        cache = {}
+
+    if not target_time:
+        return config.get_default().version
+
+    # Iterate through menu items in reverse (newest to oldest)
+    for item in reversed(config.menu):
+        if item.superuser_only:
+            continue
+        build_time = get_build_time(item.build.version, cache)
+        if build_time and build_time <= target_time:
+            return item.build.version
+
+    return None
+
+
+def get_build_time(version, cache=None):
+    if cache is None:
+        cache = {}
+
+    if version in cache:
+        return cache[version]
+
+    try:
+        build = CommCareBuild.get_build(version, latest=True)
+    except KeyError:
+        cache[version] = None
+        return None
+
+    build_time = build.time if build and build.time else None
+    cache[version] = build_time
+    return build_time
+
+
+def is_out_of_date(version_in_use, latest_version):
+    version_in_use_tuple = _parse_version(version_in_use)
+    latest_version_tuple = _parse_version(latest_version)
+    if not version_in_use_tuple or not latest_version_tuple:
+        return False
+    return version_in_use_tuple < latest_version_tuple
+
+
+def _parse_version(version_str):
+    """Convert version string to comparable tuple, padding with zeros."""
+    SEMANTIC_VERSION_PARTS = 3  # Major, minor, patch
+    if version_str:
+        try:
+            version_parts = [int(n) for n in version_str.split('.')]
+            # Pad the version tuple to ensure both versions have the same length
+            return tuple(version_parts + [0] * (SEMANTIC_VERSION_PARTS - len(version_parts)))
+        except (ValueError, AttributeError):
+            return None
+    return None

corehq/apps/callcenter/sync_usercase.py

@@ -136,10 +136,10 @@ def valid_element_name(name):
     fields = {k: v for k, v in user.get_user_data(domain).items() if
               valid_element_name(k)}
 
-    if user.is_web_user():
-        fields['commcare_location_id'] = user.get_location_id(domain)
+    if location_id := user.get_location_id(domain):
+        fields['commcare_location_id'] = location_id
         fields['commcare_location_ids'] = user_location_data(user.get_location_ids(domain))
-        fields['commcare_primary_case_sharing_id'] = user.get_location_id(domain)
+        fields['commcare_primary_case_sharing_id'] = location_id
 
     # language or phone_number can be null and will break
     # case submission
@@ -155,6 +155,7 @@ def valid_element_name(name):
         'hq_user_id': user.get_id,
         'first_name': user.first_name or '',
         'last_name': user.last_name or '',
+        'commcare_project': domain,
     })
 
     return fields

corehq/apps/cleanup/management/commands/hard_delete_forms.py

@@ -0,0 +1,56 @@
+from django.core.management.base import BaseCommand, CommandError
+import csv
+import itertools
+from dimagi.utils.chunked import chunked
+from corehq.form_processor.models import XFormInstance
+
+
+INDEX_FORM_ID = 0
+CHUNK_SIZE = 100
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('domain', help='Domain name that owns the forms to be deleted')
+        parser.add_argument('filename', help='path to the CSV file')
+        parser.add_argument('--resume_id', help='form ID to start at, within the CSV file')
+
+    def handle(self, domain, filename, resume_id=None, **options):
+        # expects the filename to have a CSV with a header containing a "Form ID" field
+        with open(filename, mode='r', encoding='utf-8-sig') as csvfile:
+            reader = csv.reader(csvfile, delimiter=',')
+            self._process_rows(reader, domain, resume_id)
+
+    def _process_rows(self, rows, domain, resume_id):
+        header_row = next(rows)   # skip header line
+        if header_row[INDEX_FORM_ID] != 'Form ID':
+            raise CommandError(
+                f'Expected Column {INDEX_FORM_ID} to be "Form ID", found "{header_row[INDEX_FORM_ID]}". Exiting'
+            )
+
+        num_deleted = 0
+
+        if resume_id:
+            print('resuming at: ', resume_id)
+            rows = itertools.dropwhile(lambda row: row[INDEX_FORM_ID] != resume_id, rows)
+
+        print('Starting form deletion')
+        for chunk in chunked(rows, CHUNK_SIZE):
+            form_ids = [row[INDEX_FORM_ID] for row in chunk]
+
+            try:
+                deleted_form_ids = set(XFormInstance.objects.hard_delete_forms(
+                    domain, form_ids, return_ids=True))
+            except Exception:
+                print('failed during processing of: ', form_ids)
+                raise
+
+            for form_id in form_ids:
+                if form_id in deleted_form_ids:
+                    print('Deleted: ', form_id)
+                else:
+                    print('Not found:', form_id)
+
+            num_deleted += len(deleted_form_ids)
+
+        print(f'Complete -- removed {num_deleted} forms')

corehq/apps/cloudcare/static/cloudcare/js/form_entry/entries.js

@@ -938,7 +938,8 @@ hqDefine("cloudcare/js/form_entry/entries", [
     FileEntry.prototype = Object.create(EntrySingleAnswer.prototype);
     FileEntry.prototype.constructor = EntrySingleAnswer;
     FileEntry.prototype.onPreProcess = function (newValue) {
-        var self = this;
+        const self = this;
+        const cachedFilename = self.question.form().fileNameCache[self.answer()];
         if (newValue === "" && self.question.filename) {
             self.question.hasAnswered = true;
             self.fileNameDisplay(self.question.filename());
@@ -948,17 +949,26 @@ hqDefine("cloudcare/js/form_entry/entries", [
                 self.question.formplayerMediaRequest = $.Deferred();
                 self.cleared = false;
             }
-            var fixedNewValue = newValue.replace(constants.FILE_PREFIX, "");
+            const fixedNewValue = newValue.replace(constants.FILE_PREFIX, "");
             self.fileNameDisplay(fixedNewValue);
             self.answer(fixedNewValue);
+        } else if (cachedFilename && newValue !== constants.NO_ANSWER) {
+            // The cached filename is only set if the file has been uploaded already and not cleared
+            // newValue is only empty initially and after clear. So this combination only happens when
+            // rebuilding the questions (after deleting a repeat group)
+            self.fileNameDisplay(cachedFilename);
         } else {
             self.onClear();
         }
     };
     FileEntry.prototype.onAnswerChange = function (newValue) {
         var self = this;
         // file has already been validated and assigned a unique id. another request should not be sent to formplayer
-        if (newValue === constants.NO_ANSWER || self.question.formplayerMediaRequest.state() === "resolved") {
+        if (newValue === constants.NO_ANSWER) {
+            return;
+        }
+        if (self.question.formplayerMediaRequest.state() === "resolved") {
+            self.question.form().fileNameCache[self.question.answer()] = self.file().name;
             return;
         }
         if (newValue !== constants.NO_ANSWER && newValue !== "") {

corehq/apps/cloudcare/static/cloudcare/js/form_entry/form_ui.js

@@ -469,6 +469,7 @@ hqDefine("cloudcare/js/form_entry/form_ui", [
         self.atLastIndex = ko.observable(false);
         self.atFirstIndex = ko.observable(true);
         self.shouldAutoSubmit = json.shouldAutoSubmit;
+        self.fileNameCache = {};
 
         var _updateIndexCallback = function (ix, isAtFirstIndex, isAtLastIndex) {
             self.currentIndex(ix.toString());

corehq/apps/cloudcare/static/cloudcare/js/form_entry/spec/form_ui_spec.js

@@ -266,6 +266,7 @@ hqDefine("cloudcare/js/form_entry/spec/form_ui_spec", [
             // simulate response processing from FP
             question.pendingAnswer(_.clone(question.answer()));
             question.formplayerMediaRequest = {state: () => "resolved"};
+            question.entry.file({name: "chucknorris.png"});
             questionJSON.answer = "autogenerated.png";
             formJSON.tree = [questionJSON];
             $.publish('session.reconcile', [_.clone(formJSON), question]);

corehq/apps/commtrack/static/commtrack/js/products_and_programs_main.js

@@ -7,6 +7,7 @@ hqDefine('commtrack/js/products_and_programs_main', [
     'es6!hqwebapp/js/bootstrap5_loader',
     'commtrack/js/base_list_view_model',
     'hqwebapp/js/bootstrap5/widgets',   // "Additional Information" on product page uses a .hqwebapp-select2
+    'hqwebapp/js/bootstrap5/knockout_bindings.ko',  // fadeVisible
     'commcarehq',
 ], function (
     $,

corehq/apps/domain/static/domain/js/internal_calculations.js

@@ -21,7 +21,7 @@ hqDefine("domain/js/internal_calculations", [
                 $btn.addClass('btn-danger');
                 $error.html(data.error);
             }
-            $btn.html('Reload Data').removeClass('btn-primary');
+            $btn.html('Reload Data').removeClass('btn-primary').addClass('btn-outline-primary');
         });
     }
 

corehq/apps/domain/templates/domain/bootstrap5/activate_transfer_domain.html → corehq/apps/domain/templates/domain/activate_transfer_domain.html

@@ -5,7 +5,7 @@
 {% block content %}
   <div class="container" id="hq-content">
     <div class="row">
-      <div class="page-header">  {# todo B5: css-page-header #}
+      <div class="mt-5">
         <h1>{% trans "Transfer project ownership" %}</h1>
       </div>
       <div class="col-lg-8 col-sm-12">