truncatesc: add new style time partitions

db/comdb2.c

@@ -6651,6 +6651,8 @@ int comdb2_reload_schemas(void *dbenv, void *inlsn)
 
     gbl_watcher_thread_ran = comdb2_time_epoch();
     unlock_schema_lk();
+    /* requeue partition rollouts now that the reloaded tables are open */
+    XCHANGE32(gbl_trigger_timepart, 1);
     logmsg(LOGMSG_INFO, "%s complete [%d:%d]\n", __func__, *file, *offset);
 
     gbl_comdb2_reload_schemas = 0;

db/cron.c

@@ -687,3 +687,33 @@ void cron_clear_queue_all(void)
     }
     Pthread_rwlock_unlock(&crons.rwlock);
 }
+
+/**
+ * Clear the queues of all schedulers of a given type
+ *
+ */
+void cron_clear_queue_type(enum cron_type type)
+{
+    cron_sched_t *sched = NULL;
+    cron_sched_t **scheds;
+    int n = 0;
+
+    /* clear without crons.rwlock: cron_lock waits on running events, which can
+     * need the schema lock; schedulers are never removed from the list */
+    Pthread_rwlock_rdlock(&crons.rwlock);
+    scheds = malloc(sizeof(cron_sched_t *) * (listc_size(&crons.scheds) + 1));
+    if (scheds) {
+        LISTC_FOR_EACH(&crons.scheds, sched, lnk)
+        {
+            if (sched->impl.type == type)
+                scheds[n++] = sched;
+        }
+    } else {
+        logmsg(LOGMSG_ERROR, "%s: malloc failed, queues not cleared\n", __func__);
+    }
+    Pthread_rwlock_unlock(&crons.rwlock);
+
+    for (int i = 0; i < n; i++)
+        cron_clear_queue(scheds[i]);
+    free(scheds);
+}

db/cron.h

@@ -157,6 +157,12 @@ void cron_signal_all(void);
  */
 void cron_clear_queue_all(void);
 
+/**
+ * Clear the queues of all schedulers of a given type
+ *
+ */
+void cron_clear_queue_type(enum cron_type type);
+
 /**
  * Returns a scheduler with name "name", if any
  * NOTE: scheduler is not locked

db/logical_cron.c

@@ -50,11 +50,16 @@ static int logical_wait_next_event(sched_if_t *impl, cron_event_t *event)
 
     rc = cron_timedwait(sched, &ts);
 
-    /* refresh the counter here */
-    state->clock = logical_cron_read_persistent(impl->name, &err);
+    /* refresh the counter without sched->mtx: the read takes the schema lock,
+     * and schema-locked paths take sched->mtx; keep the old clock on error */
+    cron_unlock(sched);
+    unsigned long long clock = logical_cron_read_persistent(impl->name, &err);
+    cron_lock(sched);
     if (errstat_get_rc(&err) != VIEW_NOERR) {
         logmsg(LOGMSG_ERROR, "%s error rc %d \"%s\"\n", __func__,
                errstat_get_rc(&err), errstat_get_str(&err));
+    } else {
+        state->clock = clock;
     }
 
     return rc;
@@ -109,7 +114,6 @@ static void *logical_cron_kickoff(struct cron_event *event, struct errstat *err)
     sched_if_t *schedif = event->schedif;
     logical_state_t *state = schedif->state;
     struct sql_thread *thd;
-    const char *tablename = LOGICAL_CRON_SYSTABLE;
 
     logmsg(LOGMSG_INFO, "Starting logical cron %s\n", (char *)event->arg1);
 
@@ -121,9 +125,6 @@ static void *logical_cron_kickoff(struct cron_event *event, struct errstat *err)
         return NULL;
     }
 
-    /* construct rootpage cache that includes only logical_cron_systable */
-    get_copy_rootpages_selectfire(thd, 1, &tablename, NULL, NULL, 1);
-
     /* check to see if there is a persistent value for this scheduler */
     state->clock = logical_cron_read_persistent(schedif->name, err);
     if (errstat_get_rc(err) != VIEW_NOERR) {
@@ -212,7 +213,7 @@ unsigned long long logical_cron_read_persistent(const char *name,
     }
 
     bdb_thread_event(thedb->bdb_env, BDBTHR_EVENT_START);
-    counter = run_sql_thd_return_ll(query, thd, err);
+    counter = run_sql_thd_return_ll(query, thd, LOGICAL_CRON_SYSTABLE, err);
     if (counter == LLONG_MIN)
         counter = 0;
     bdb_thread_event(thedb->bdb_env, BDBTHR_EVENT_DONE);

db/osqlcomm.c

@@ -7631,7 +7631,15 @@ static int _process_partition_retention(struct ireq *iq)
     sc->nothrevent = 1;    /* serialize, mirrors create/rollout paths */
 
     rc = SC_OK;
-    for (i = 0; i < nnames; i++) {
+    if (is_increase) {
+        /* new shards inherit the partition's table version */
+        sc->timepartition_version = comdb2_table_version(sc->tablename);
+        if (sc->timepartition_version == (unsigned long long)-1) {
+            sc_errf(sc, "Failed to get the version of partition %s", sc->tablename);
+            rc = ERR_SC;
+        }
+    }
+    for (i = 0; rc == SC_OK && i < nnames; i++) {
         struct schema_change_type *cur;
 
         if (i == 0) {

db/sql.h

@@ -1699,8 +1699,7 @@ void add_fingerprint(struct sqlclntstate *, sqlite3_stmt *, struct string_ref *,
                      int64_t, int64_t, struct reqlogger *, unsigned char *, int);
 
 long long run_sql_return_ll(const char *query, struct errstat *err);
-long long run_sql_thd_return_ll(const char *query, struct sql_thread *thd,
-                                struct errstat *err);
+long long run_sql_thd_return_ll(const char *query, struct sql_thread *thd, const char *locktbl, struct errstat *err);
 
 struct query_plan_item {
     unsigned char plan_fingerprint[FINGERPRINTSZ]; /* md5 digest hex string */

db/sqlglue.c

@@ -14057,14 +14057,14 @@ long long run_sql_return_ll(const char *sql, struct errstat *err)
     int rc;
 
     thd = start_sql_thread();
-    rc = run_sql_thd_return_ll(sql, thd, err);
+    rc = run_sql_thd_return_ll(sql, thd, NULL, err);
     done_sql_thread();
 
     return rc;
 }
 
-long long run_sql_thd_return_ll(const char *query, struct sql_thread *thd,
-                                struct errstat *err)
+/* if locktbl is set, read it under the schema lock with fresh rootpages */
+long long run_sql_thd_return_ll(const char *query, struct sql_thread *thd, const char *locktbl, struct errstat *err)
 {
     sqlite3 *sqldb;
     int rc;
@@ -14088,6 +14088,15 @@ long long run_sql_thd_return_ll(const char *query, struct sql_thread *thd,
         goto done;
     }
 
+    if (locktbl) {
+        clnt.in_sqlite_init = 1;
+        rdlock_schema_lk();
+        if ((rc = get_copy_rootpages_selectfire(thd, 1, &locktbl, NULL, NULL, 0)) != 0) {
+            errstat_set_rcstrf(err, -1, "%s: failed to get rootpages rc %d", __func__, rc);
+            goto cleanup;
+        }
+    }
+
     if ((rc = sqlite3_open_serial("db", &sqldb, NULL)) != 0) {
         errstat_set_rcstrf(err, -1, "%s:sqlite3_open_serial rc %d", __func__,
                            rc);
@@ -14104,6 +14113,10 @@ long long run_sql_thd_return_ll(const char *query, struct sql_thread *thd,
         errstat_set_rcstrf(err, -1, "close rc %d\n", crc);
 
 cleanup:
+    if (locktbl) {
+        unlock_schema_lk();
+        clnt.in_sqlite_init = 0;
+    }
     crc = put_curtran(thedb->bdb_env, &clnt);
     if (crc && !rc)
         errstat_set_rcstrf(err, -1, "%s: failed to close curtran", __func__);

db/views.c

@@ -199,17 +199,19 @@ timepart_views_t *timepart_views_init(struct dbenv *dbenv)
     struct errstat xerr = {0};
     const char *name = "timepart_cron";
 
-    time_cron_create(&tpt_cron, timepart_describe, timepart_event_describe);
+    /* a schema reload keeps the lock and scheduler; it requeues the events */
+    if (!timepart_sched) {
+        time_cron_create(&tpt_cron, timepart_describe, timepart_event_describe);
 
-    Pthread_rwlock_init(&views_lk, NULL);
+        Pthread_rwlock_init(&views_lk, NULL);
 
-    /* create the timepart scheduler */
-    timepart_sched = cron_add_event(NULL, name, INT_MIN, timepart_cron_kickoff,
-                                    NULL, NULL, NULL, NULL, NULL, &xerr,
-                                    &tpt_cron);
+        /* create the timepart scheduler */
+        timepart_sched =
+            cron_add_event(NULL, name, INT_MIN, timepart_cron_kickoff, NULL, NULL, NULL, NULL, NULL, &xerr, &tpt_cron);
 
-    if (timepart_sched == NULL)
-        return NULL;
+        if (timepart_sched == NULL)
+            return NULL;
+    }
 
     /* read the llmeta views, if any */
     dbenv->timepart_views = views_create_all_views();
@@ -1952,6 +1954,8 @@ int views_cron_restart(timepart_views_t *views)
     /* in case of regular master swing, clear pre-existing views event,
        we will requeue them */
     cron_clear_queue(timepart_sched);
+    /* also partitions removed by a schema reload */
+    cron_clear_queue_type(CRON_LOGICAL);
 
     /* Acquire BDB replock before views_lk to match the ordering used in
        _view_cron_phase1/3 (BDB_READLOCK -> views_lk).  Consistent ordering
@@ -2781,8 +2785,8 @@ int timepart_validate_retention_change(const char *name, int new_retention, int
  *
  * The shards are walked in rollout order starting right after the current
  * shard (i.e. oldest-next-to-be-recycled first), ending with the current
- * shard itself. The reconfigured ring is built so that the (unchanged)
- * current shard always ends up at the last index:
+ * shard itself. The reconfigured ring is built with the (unchanged) current
+ * shard at the last index, then rotated so the old shard 0 stays at index 0:
  *   - increase: new empty shards are prepended (consumed next, preserving
  *     history), followed by the old shards in rollout order.
  *   - decrease: the oldest shards are dropped from the front of the rollout
@@ -2855,9 +2859,8 @@ int timepart_reconfigure_retention(const char *name, int new_retention, int is_m
         goto done_locked;
     }
 
-    /* NOTE: current_shard is not persisted, it is recomputed on reload by
-     * _view_find_current_shard(); that lookup ignores never-used (INT_MAX)
-     * shards, so the array order below does not affect the ring position. */
+    /* current_shard is recomputed on reload ignoring unused shards, so the
+     * rotation below that keeps the original table at index 0 is safe */
 
     if (is_increase) {
         int nnew = new_retention - old_retention;
@@ -2936,6 +2939,24 @@ int timepart_reconfigure_retention(const char *name, int new_retention, int is_m
         nnames = ndrop;
     }
 
+    /* rotate the ring so the old shard 0 (possibly aliased) stays at index 0 */
+    for (i = 0; i < new_retention; i++) {
+        if (strcasecmp(newview->shards[i].tblname, view->shards[0].tblname) == 0)
+            break;
+    }
+    if (i > 0 && i < new_retention) {
+        timepart_shard_t *rotated = malloc(sizeof(timepart_shard_t) * new_retention);
+        if (!rotated) {
+            errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__);
+            goto done_locked;
+        }
+        for (int j = 0; j < new_retention; j++)
+            rotated[j] = newview->shards[(j + i) % new_retention];
+        memcpy(newview->shards, rotated, sizeof(timepart_shard_t) * new_retention);
+        free(rotated);
+        newview->current_shard = new_retention - 1 - i;
+    }
+
     rc = VIEW_NOERR;
 
 done_locked:

tests/quarantine.csv

@@ -17,7 +17,6 @@ insert_lots,UNKNOWN,180779489
 insert_lots_ssl_generated,UNKNOWN,180779489
 incoh_remsql,UNKNOWN,180788122
 incoh_remsql_rte_connect_generated,UNKNOWN,180788122
-truncatesc,DB_BUG,180801974
 truncatesc_offline_generated,DB_BUG,180801974
 sc_partial_datacopy_logicalsc_generated,UNKNOWN,180802210
 lock_stat_inversion,UNKNOWN,180915326

tests/truncatesc.test/Makefile

@@ -4,5 +4,5 @@ else
   include $(TESTSROOTDIR)/testcase.mk
 endif
 ifeq ($(TEST_TIMEOUT),)
-	export TEST_TIMEOUT=40m
+	export TEST_TIMEOUT=50m
 endif