feat: Adds functionality to clear out bad shard list

tsdb/store.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -672,6 +673,34 @@ func (s *Store) Shard(id uint64) *Shard {
 	return sh
 }
 
+// ClearBadShardList will remove all shards from the badShards cache
+// this will allow for lazy loading of bad shards if/when they are no
+// longer in a "bad" state. This method will return any shards that
+// were removed from the cache.
+func (s *Store) ClearBadShardList() map[uint64]error {
+	s.badShards.mu.Lock()
+	defer s.badShards.mu.Unlock()
+
+	if s.badShards.shardErrors == nil {
+		s.Logger.Warn("badShards was nil")
+		s.badShards.shardErrors = make(map[uint64]error)
+	}
+	badShards := maps.Clone(s.badShards.shardErrors)
+	clear(s.badShards.shardErrors)
+
+	return badShards
+}
+
+// GetBadShardList is exposed as a method for test purposes
+func (s *Store) GetBadShardList() map[uint64]error {
+	s.badShards.mu.Lock()
+	defer s.badShards.mu.Unlock()
+
+	shardList := maps.Clone(s.badShards.shardErrors)
+
+	return shardList
+}
+
 type ErrPreviousShardFail struct {
 	error
 }

tsdb/store_test.go

@@ -262,6 +262,70 @@ func TestStore_BadShard(t *testing.T) {
 	}
 }
 
+func TestStore_BadShardClear(t *testing.T) {
+	const errStr = "a shard open error"
+	indexes := tsdb.RegisteredIndexes()
+	for _, idx := range indexes {
+		func() {
+			s := MustOpenStore(t, idx)
+			defer require.NoErrorf(t, s.Close(), "closing store with index type: %s", idx)
+
+			sh := tsdb.NewTempShard(idx)
+			shId := sh.ID()
+			err := s.OpenShard(sh.Shard, false)
+			require.NoError(t, err, "opening temp shard")
+			defer require.NoError(t, sh.Close(), "closing temporary shard")
+
+			expErr := errors.New(errStr)
+			s.SetShardOpenErrorForTest(sh.ID(), expErr)
+			err2 := s.OpenShard(sh.Shard, false)
+			require.Error(t, err2, "no error opening bad shard")
+			require.ErrorIs(t, err2, tsdb.ErrPreviousShardFail{})
+			require.EqualError(t, err2, fmt.Errorf("not attempting to open shard %d; opening shard previously failed with: %w", shId, expErr).Error())
+
+			require.Equal(t, 1, len(s.Store.GetBadShardList()))
+
+			badShards := s.ClearBadShardList()
+			require.Len(t, badShards, 1)
+
+			// Check that bad shard list has been cleared
+			require.Empty(t, s.Store.GetBadShardList())
+
+			s.SetShardOpenErrorForTest(sh.ID(), expErr)
+			err2 = s.OpenShard(sh.Shard, false)
+			require.Error(t, err2, "no error opening bad shard")
+			require.ErrorIs(t, err2, tsdb.ErrPreviousShardFail{})
+			require.EqualError(t, err2, fmt.Errorf("not attempting to open shard %d; opening shard previously failed with: %w", shId, expErr).Error())
+
+			// Check that bad shard list now has a bad shard in it
+			require.Len(t, s.Store.GetBadShardList(), 1)
+		}()
+	}
+}
+
+func TestStore_BadShardClearNoBadShards(t *testing.T) {
+	indexes := tsdb.RegisteredIndexes()
+	for _, idx := range indexes {
+		func() {
+			s := MustOpenStore(t, idx)
+			defer require.NoErrorf(t, s.Close(), "closing store with index type: %s", idx)
+
+			sh := tsdb.NewTempShard(idx)
+			err := s.OpenShard(sh.Shard, false)
+			require.NoError(t, err, "opening temp shard")
+			defer require.NoError(t, sh.Close(), "closing temporary shard")
+
+			require.Empty(t, s.Store.GetBadShardList())
+
+			badShards := s.ClearBadShardList()
+			require.Empty(t, badShards)
+
+			// Check that bad shard list has been cleared
+			require.Empty(t, s.Store.GetBadShardList())
+		}()
+	}
+}
+
 func TestStore_CreateMixedShards(t *testing.T) {
 	t.Parallel()