Thanos源码专题【左扬精讲】——Thanos Compact 组件(release-0.26)源码阅读和分析(详解 cmd/compact.go )

Thanos Compact 组件(release-0.26)源码阅读和分析(详解 cmd/compact.go )

https://github.com/thanos-io/thanos/blob/v0.26.0/cmd/thanos/compact.go

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
// Copyright (c) The Thanos Authors.
// Licensed under the Apache License 2.0.
 
package main
 
import (
    "context"
    "fmt"
    "os"
    "path"
    "strconv"
    "strings"
    "sync"
    "time"
 
    "github.com/alecthomas/units"
    extflag "github.com/efficientgo/tools/extkingpin"
    "github.com/go-kit/log"
    "github.com/go-kit/log/level"
    "github.com/oklog/run"
    "github.com/opentracing/opentracing-go"
    "github.com/pkg/errors"
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promauto"
    "github.com/prometheus/common/model"
    "github.com/prometheus/common/route"
    "github.com/prometheus/prometheus/storage"
    "github.com/prometheus/prometheus/tsdb"
 
    blocksAPI "github.com/thanos-io/thanos/pkg/api/blocks"
    "github.com/thanos-io/thanos/pkg/block"
    "github.com/thanos-io/thanos/pkg/block/metadata"
    "github.com/thanos-io/thanos/pkg/compact"
    "github.com/thanos-io/thanos/pkg/compact/downsample"
    "github.com/thanos-io/thanos/pkg/component"
    "github.com/thanos-io/thanos/pkg/dedup"
    "github.com/thanos-io/thanos/pkg/extkingpin"
    "github.com/thanos-io/thanos/pkg/extprom"
    extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http"
    "github.com/thanos-io/thanos/pkg/logging"
    "github.com/thanos-io/thanos/pkg/objstore/client"
    "github.com/thanos-io/thanos/pkg/prober"
    "github.com/thanos-io/thanos/pkg/runutil"
    httpserver "github.com/thanos-io/thanos/pkg/server/http"
    "github.com/thanos-io/thanos/pkg/store"
    "github.com/thanos-io/thanos/pkg/tracing"
    "github.com/thanos-io/thanos/pkg/ui"
)
 
var (
    // 默认的compactions是:1小时,2小时,8小时,2天,14天。
    compactions = compactionSet{
        1 * time.Hour,
        2 * time.Hour,
        8 * time.Hour,
        2 * 24 * time.Hour,
        14 * 24 * time.Hour,
    }
)
 
type compactionSet []time.Duration
 
// String 方法返回 compactionSet 的字符串表示形式。
// 该字符串是一个逗号分隔的字符串,每个元素都表示为 "i=ch" 的形式,
// 其中 i 是 compactionSet 的索引,ch 是 compactionSet 中对应元素的小时数。
func (cs compactionSet) String() string {
    // 创建一个字符串切片,长度为 compactionSet 的长度
    result := make([]string, len(cs))
    // 遍历 compactionSet
    for i, c := range cs {
        // 将每个元素格式化为 "i=ch" 的形式,其中 c.Hours() 表示小时数。
        result[i] = fmt.Sprintf("%d=%dh", i, int(c.Hours()))
    }
    // 使用逗号加空格将结果切片中的元素连接起来。返回结果:["0=1h", "1=2h", "2=8h", "3=48h", "4=336h"]
    return strings.Join(result, ", ")
}
 
// levels returns set of compaction levels not higher than specified max compaction level.
// levels 函数接收一个 compactionSet 类型的参数 cs 和一个 int 类型的参数 maxLevel,
// 返回一个 []int64 类型的切片和一个 error 类型的错误。
// 该函数的功能是返回给定 compactionSet 集合中前 maxLevel 个级别的值,
// 每个级别的值以毫秒为单位转换为 int64 类型。
//
// 如果 maxLevel 大于等于 cs 的长度,则返回一个 nil 切片和一个错误信息,
// 错误信息为 "level is bigger then default set of %d",其中 %d 为 cs 的长度。
//
// 如果 maxLevel 小于 cs 的长度,则创建一个长度为 maxLevel+1 的切片 levels,
// 用于存储每个级别的值。然后遍历 cs[:maxLevel+1] 范围内的元素,
// 将每个元素除以 time.Millisecond,然后转换为 int64 类型后存储到 levels 中。
//
// 最后,返回存储级别值的切片 levels 和 nil 错误。
func (cs compactionSet) levels(maxLevel int) ([]int64, error) {
    // 如果 maxLevel 大于等于 cs 的长度,则返回错误
    if maxLevel >= len(cs) {
        return nil, errors.Errorf("level is bigger then default set of %d", len(cs))
    }
 
    // 创建一个长度为 maxLevel+1 的切片,用于存储每个级别的值
    levels := make([]int64, maxLevel+1)
    // 遍历 cs[:maxLevel+1] 范围内的元素,并将每个元素转换为 int64 类型后存储到 levels 中
    for i, c := range cs[:maxLevel+1] {
        // 将每个元素除以 time.Millisecond,然后转换为 int64 类型
        levels[i] = int64(c / time.Millisecond)
    }
    // 返回存储级别值的切片和 nil 错误
    return levels, nil
}
 
// maxLevel returns max available compaction level.
// maxLevel 返回 compactionSet 的最大层级的索引。
//
// 该函数返回 compactionSet 的长度减一,即最大层级的索引。
func (cs compactionSet) maxLevel() int {
    // 返回 compactionSet 的长度减一,即最大层级的索引
    return len(cs) - 1
}
 
func registerCompact(app *extkingpin.App) {
    // 创建一个命令对象 cmd,用于执行压缩操作
    cmd := app.Command(component.Compact.String(), "Continuously compacts blocks in an object store bucket.")
 
    // 创建一个压缩配置对象 conf
    conf := &compactConfig{}
    // 注册命令行参数到 conf 中,包括对象存储配置、压缩级别等参数
    conf.registerFlag(cmd)
 
    // 设置命令执行前的准备工作函数,该函数在命令执行前被调用,用于设置命令的参数和参数解析器
    cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error {
        // 执行压缩操作
        return runCompact(g, logger, tracer, reg, component.Compact, *conf, getFlagsMap(cmd.Flags()))
    })
}
 
type compactMetrics struct {
    halted                      prometheus.Gauge       // halted 是一个 Gauge 类型的指标,用于表示压缩操作是否已停止
    retried                     prometheus.Counter     // retried 是一个 Counter 类型的指标,用于记录可重试的压缩器错误的总次数
    iterations                  prometheus.Counter     // iterations 是一个 Counter 类型的指标,用于记录压缩操作的总次数
    cleanups                    prometheus.Counter     // cleanups 是一个 Counter 类型的指标,用于记录并发清理循环的总次数
    partialUploadDeleteAttempts prometheus.Counter     // partialUploadDeleteAttempts 是一个 Counter 类型的指标,用于记录已尝试删除的假定中止且仅部分上传的块的总次数
    blocksCleaned               prometheus.Counter     // blocksCleaned 是一个 Counter 类型的指标,用于记录已清理的块的总次数
    blockCleanupFailures        prometheus.Counter     // blockCleanupFailures 是一个 Counter 类型的指标,用于记录块清理失败的次数
    blocksMarked                *prometheus.CounterVec // blocksMarked 是一个 CounterVec 类型的指标,用于记录标记的块的总次数
    garbageCollectedBlocks      prometheus.Counter     // garbageCollectedBlocks 是一个 Counter 类型的指标,用于记录垃圾收集的块的总次数
}
 
// newCompactMetrics 创建一个新的compactMetrics对象,用于跟踪和记录Thanos Compactor 组件的性能指标。
// 参数:
//   - reg: *prometheus.Registry,Prometheus注册器,用于注册指标。
//   - deleteDelay: time.Duration,配置的删除延迟时间。
//
// 返回值:
//   - *compactMetrics,包含多个Prometheus指标的compactMetrics对象。
func newCompactMetrics(reg *prometheus.Registry, deleteDelay time.Duration) *compactMetrics {
    /*
        这段代码创建并注册了一个名为"thanos_delete_delay_seconds"的Prometheus Gauge指标,该指标描述了配置的删除延迟(以秒为单位)。每次Prometheus抓取指标时,它都会调用提供的函数来获取当前的删除延迟值(以秒为单位)。这使得Prometheus能够监控和记录这个值的变化,进而用于监控和警报等目的。
 
        promauto.With(reg).NewGaugeFunc:
            promauto是Prometheus客户端库中的一个包,它提供了自动注册指标的便捷方法。With(reg)是一个方法调用,其中reg是一个prometheus.Registerer接口的实现,通常是一个prometheus.Registry实例。这个方法返回一个配置了对象,允许你创建并自动注册指标。
            NewGaugeFunc是一个方法,用于创建一个新的Gauge类型的指标。Gauge指标用于表示可以任意上下浮动的数值,例如当前内存使用量、队列长度等。与普通的Gauge不同,NewGaugeFunc接受一个函数作为参数,这个函数在每次Prometheus抓取指标时被调用以获取当前的数值。
 
        prometheus.GaugeOpts:
            GaugeOpts是一个结构体,用于配置Gauge指标的名称、帮助信息等属性。它通常作为NewGaugeFunc方法的参数之一传入,以定义新创建的Gauge指标的具体行为和属性。
            Name:指标的名称,这里设置为"thanos_delete_delay_seconds"。
            Help:关于这个指标的描述,这里设置为"Configured delete delay in seconds."。
    */
    _ = promauto.With(reg).NewGaugeFunc(prometheus.GaugeOpts{
        Name: "thanos_delete_delay_seconds",
        Help: "Configured delete delay in seconds.",
    }, func() float64 { // func() float64 { return deleteDelay.Seconds() }是一个匿名函数,它不接受参数并返回一个float64类型的值。这个函数在每次Prometheus抓取指标时被调用。
        return deleteDelay.Seconds() // Seconds()方法将这个持续时间转换为秒,返回的秒数作为指标的当前值。
    })
 
    m := &compactMetrics{}
 
    /*
        创建一个新的Gauge指标,用于表示Compactor是否因意外错误而停止。
 
        使用promauto包(Prometheus自动注册工具)来创建一个新的Gauge指标。
 
        promauto.With(reg).NewGauge函数接受一个prometheus.GaugeOpts结构体作为参数,该结构体定义了指标的名称(Name)和帮助信息(Help)。
        在这个例子中,指标的名称是"thanos_compact_halted",帮助信息是"Set to 1 if the compactor halted due to an unexpected error.",意思是如果Compactor因意外错误而停止,则将该指标设置为1。
    */
    m.halted = promauto.With(reg).NewGauge(prometheus.GaugeOpts{
        Name: "thanos_compact_halted",
        Help: "Set to 1 if the compactor halted due to an unexpected error.",
    })
    // 将新创建的Gauge指标的初始值设置为0。这意味着在 Compactor 正常运行时,该指标应该保持为0。
    m.halted.Set(0)
 
    // 创建一个新的Counter指标,用于表示Compactor在可重试错误后重试的总次数
    m.retried = promauto.With(reg).NewCounter(prometheus.CounterOpts{
        Name: "thanos_compact_retries_total",
        Help: "Total number of retries after retriable compactor error.",
    })
 
    // 创建一个新的Counter指标,用于表示成功执行的迭代次数
    m.iterations = promauto.With(reg).NewCounter(prometheus.CounterOpts{
        Name: "thanos_compact_iterations_total",
        Help: "Total number of iterations that were executed successfully.",
    })
 
    // 创建一个新的Counter指标,用于表示成功执行的并发清理循环次数
    m.cleanups = promauto.With(reg).NewCounter(prometheus.CounterOpts{
        Name: "thanos_compact_block_cleanup_loops_total",
        Help: "Total number of concurrent cleanup loops of partially uploaded blocks and marked blocks that were executed successfully.",
    })
 
    // 创建一个新的Counter指标,用于表示启动删除假设已中止且仅部分上传的块的总次数
    m.partialUploadDeleteAttempts = promauto.With(reg).NewCounter(prometheus.CounterOpts{
        Name: "thanos_compact_aborted_partial_uploads_deletion_attempts_total",
        Help: "Total number of started deletions of blocks that are assumed aborted and only partially uploaded.",
    })
 
    // 创建一个新的Counter指标,用于表示在Compactor中删除的块的总数
    m.blocksCleaned = promauto.With(reg).NewCounter(prometheus.CounterOpts{
        Name: "thanos_compact_blocks_cleaned_total",
        Help: "Total number of blocks deleted in compactor.",
    })
 
    // 创建一个新的Counter指标,用于表示在Compactor中删除块时遇到的失败次数
    m.blockCleanupFailures = promauto.With(reg).NewCounter(prometheus.CounterOpts{
        Name: "thanos_compact_block_cleanup_failures_total",
        Help: "Failures encountered while deleting blocks in compactor.",
    })
 
    // 创建一个新的CounterVec指标,用于表示在Compactor中标记的块的总数,带有标签"marker"和"reason"
    m.blocksMarked = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
        Name: "thanos_compact_blocks_marked_total",
        Help: "Total number of blocks marked in compactor.",
    }, []string{"marker", "reason"})
    // 设置标记的块标签值
    m.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.OutOfOrderChunksNoCompactReason)
    m.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.IndexSizeExceedingNoCompactReason)
    m.blocksMarked.WithLabelValues(metadata.DeletionMarkFilename, "")
 
    // 创建一个新的Counter指标,用于表示Compactor标记为删除的块的总数
    m.garbageCollectedBlocks = promauto.With(reg).NewCounter(prometheus.CounterOpts{
        Name: "thanos_compact_garbage_collected_blocks_total",
        Help: "Total number of blocks marked for deletion by compactor.",
    })
    return m
}
 
// runCompact 是启动和管理 Thanos 组件压缩过程的主要函数。
// 它配置压缩器,设置指标,初始化 HTTP 服务器,并管理压缩过程的生命周期。
//
// 参数:
// - g: 用于管理压缩过程生命周期的 run.Group。
// - logger: 用于记录信息和错误的日志记录器。
// - tracer: 用于跟踪操作的 opentracing.Tracer。
// - reg: 用于注册指标的 prometheus.Registry。
// - component: 此压缩器管理的组件。
// - conf: 包含压缩器配置的 compactConfig。
// - flagsMap: 包含附加配置的标志名称到其值的映射。
//
// 返回值:
// - rerr: 如果在设置或运行压缩器时出现问题,则返回一个错误,否则返回 nil。
func runCompact(
    g *run.Group, // 用于管理压缩过程生命周期的 run.Group。
    logger log.Logger, // 用于记录信息和错误的日志记录器。
    tracer opentracing.Tracer, // 用于跟踪操作的 opentracing.Tracer。
    reg *prometheus.Registry, // 用于注册指标的 prometheus.Registry。
    component component.Component, // 此压缩器管理的组件。
    conf compactConfig, // 包含压缩器配置的 compactConfig。
    flagsMap map[string]string, // 包含附加配置的标志名称到其值的映射。
) (rerr error) {
    deleteDelay := time.Duration(conf.deleteDelay)        // 删除延迟时间
    compactMetrics := newCompactMetrics(reg, deleteDelay) // 创建并初始化用于跟踪压缩性能指标的 compactMetrics 对象。
    downsampleMetrics := newDownsampleMetrics(reg)        // 创建并初始化用于跟踪下采样性能指标的 downsampleMetrics 对象。
 
    // 创建一个用于管理压缩过程的压缩器对象。
    httpProbe := prober.NewHTTP()
    // Combine 方法用于组合多个探针为一个单一的探针。这里将 httpProbe 和一个用于监控指标注册表的探针组合在一起。
    statusProber := prober.Combine(
        httpProbe,
        prober.NewInstrumentation(component, logger, extprom.WrapRegistererWithPrefix("thanos_", reg)),
    )
 
    // httpserver.New 创建一个新的 HTTP 服务器的实例。
    srv := httpserver.New(
        logger,    // 用于记录 HTTP 服务器相关信息的日志记录器。
        reg,       // 用于注册 HTTP 服务器指标的 prometheus.Registry。
        component, // 此 HTTP 服务器管理的组件。
        httpProbe, // 用于监控 HTTP 请求的探针。
        httpserver.WithListen(conf.http.bindAddress),                     // 配置 HTTP 服务器的监听地址。
        httpserver.WithGracePeriod(time.Duration(conf.http.gracePeriod)), // 配置 HTTP 服务器的优雅关闭时间。
        httpserver.WithTLSConfig(conf.http.tlsConfig),                    // 配置 HTTP 服务器的 TLS 配置。
    )
 
    g.Add(func() error {
        // 启动 HTTP 服务器。
        statusProber.Healthy()
 
        return srv.ListenAndServe() // 返回 HTTP 服务器的错误。
    }, func(err error) { // 当 HTTP 服务器启动失败或关闭时执行的回调函数。
        // 当 HTTP 服务器启动失败时,将探针状态设置为不可用。
        statusProber.NotReady(err)
        // 当 HTTP 服务器关闭时,将探针状态设置为不可用。
        defer statusProber.NotHealthy(err)
        // 关闭 HTTP 服务器的优雅方式。
        srv.Shutdown(err)
    })
 
    // confContentYaml 是一个包含对象存储配置的 YAML 字符串。 objStore 是一个包含对象存储配置的对象。
    confContentYaml, err := conf.objStore.Content()
    if err != nil {
        return err
    }
 
    // client.NewBucket 创建一个新的对象存储客户端。
    bkt, err := client.NewBucket(logger, confContentYaml, reg, component.String())
    if err != nil {
        return err
    }
 
    // selectorRelabelConf 是一个包含对象存储选择器重新标签配置的对象。
    relabelContentYaml, err := conf.selectorRelabelConf.Content()
    if err != nil {
        return errors.Wrap(err, "get content of relabel configuration")
    }
    // ParseRelabelConfig 解析对象存储选择器重新标签配置。
    relabelConfig, err := block.ParseRelabelConfig(relabelContentYaml, block.SelectorSupportedRelabelActions)
    if err != nil {
        return err
    }
 
    // Ensure we close up everything properly.
    // 确保在发生错误时正确关闭对象存储客户端。
    defer func() {
        if err != nil {
            runutil.CloseWithLogOnErr(logger, bkt, "bucket client")
        }
    }()
 
    // While fetching blocks, we filter out blocks that were marked for deletion by using IgnoreDeletionMarkFilter.
    // The delay of deleteDelay/2 is added to ensure we fetch blocks that are meant to be deleted but do not have a replacement yet.
    // This is to make sure compactor will not accidentally perform compactions with gap instead.
    // 当获取块时,我们使用 IgnoreDeletionMarkFilter 过滤掉被标记为删除的块。延迟删除延迟的一半是为了确保我们获取了将被删除但还没有替换的块。这是为了确保压缩器不会意外地执行带有间隙的压缩。
    // ignoreDeletionMarkFilter 是一个用于过滤删除标记的对象。用于过滤删除标记的块。
    ignoreDeletionMarkFilter := block.NewIgnoreDeletionMarkFilter(logger, bkt, deleteDelay/2, conf.blockMetaFetchConcurrency)
    // duplicateBlocksFilter 是一个用于过滤重复块的对象。
    duplicateBlocksFilter := block.NewDeduplicateFilter(conf.blockMetaFetchConcurrency)
    // noCompactMarkerFilter 是一个用于过滤不需要压缩的块的对象。
    noCompactMarkerFilter := compact.NewGatherNoCompactionMarkFilter(logger, bkt, conf.blockMetaFetchConcurrency)
    // labelShardedMetaFilter 是一个用于过滤标签分片的元数据对象。
    labelShardedMetaFilter := block.NewLabelShardedMetaFilter(relabelConfig)
    // consistencyDelayMetaFilter 是一个用于过滤一致性延迟的元数据对象。
    consistencyDelayMetaFilter := block.NewConsistencyDelayMetaFilter(logger, conf.consistencyDelay, extprom.WrapRegistererWithPrefix("thanos_", reg))
    // timePartitionMetaFilter 是一个用于过滤时间分区的元数据对象。
    timePartitionMetaFilter := block.NewTimePartitionMetaFilter(conf.filterConf.MinTime, conf.filterConf.MaxTime)
 
    // baseMetaFetcher 是一个用于获取块元数据的对象。
    baseMetaFetcher, err := block.NewBaseFetcher(logger, conf.blockMetaFetchConcurrency, bkt, "", extprom.WrapRegistererWithPrefix("thanos_", reg))
 
    if err != nil {
        // 如果创建 baseMetaFetcher 失败,则返回错误。
        return errors.Wrap(err, "create meta fetcher")
    }
 
    // enableVerticalCompaction 是一个布尔值,表示是否启用垂直压缩。如果指定了 dedupReplicaLabels,则启用垂直压缩。
    // 垂直压缩是什么?
    // 垂直压缩是指将相同时间戳的相同标签的样本数据压缩在一起,这样可以减少存储空间和提高查询效率。在 Thanos 中,垂直压缩是通过将相同时间戳的样本数据合并到同一个块中来实现的。这样可以减少存储空间的使用,并提高查询效率,因为相同的标签只需要存储一次。
    // 为什么不默认开启垂直压缩?
    // 垂直压缩会增加压缩和解压缩的开销,因此默认情况下不启用。如果需要启用垂直压缩,可以通过配置文件或命令行参数来指定。如果不需要启用垂直压缩,可以省略配置项或将其设置为 false。
    enableVerticalCompaction := conf.enableVerticalCompaction
    // 如果指定了 dedupReplicaLabels,则启用垂直压缩。这是因为 dedupReplicaLabels 用于去重复制的标签,而垂直压缩可以将相同时间戳和标签的样本数据合并到同一个块中,从而减少存储空间的使用。因此,如果指定了 dedupReplicaLabels,则启用垂直压缩可以更好地利用 Thanos 的去重复制功能。
    if len(conf.dedupReplicaLabels) > 0 {
        enableVerticalCompaction = true
        level.Info(logger).Log(
            "msg", "deduplication.replica-label specified, enabling vertical compaction", "dedupReplicaLabels", strings.Join(conf.dedupReplicaLabels, ","),
        )
    }
    if enableVerticalCompaction { // 如果启用了垂直压缩,则打印日志信息。
        level.Info(logger).Log(
            "msg", "vertical compaction is enabled", "compact.enable-vertical-compaction", fmt.Sprintf("%v", conf.enableVerticalCompaction),
        )
    }
    var (
        // bkt 是一个 Bucket 客户端,用于与对象存储交互。
        // blocksAPI.NewBlocksAPI 创建一个新的 Blocks API,用于处理块相关的请求。
        // conf.label 是一个字符串,表示要过滤的标签。
        // api 是一个 BlocksAPI 客户端,用于处理块相关的请求。
        api = blocksAPI.NewBlocksAPI(logger, conf.webConf.disableCORS, conf.label, flagsMap, bkt)
        // sy 是一个 Syncer,用于同步块元数据。
        sy *compact.Syncer
    )
    {
        // Make sure all compactor meta syncs are done through Syncer.SyncMeta for readability.
        // 确保 Syncer.SyncMeta 用于同步块元数据,以提高可读性。
        // 创建一个新的 MetaFetcher,用于获取块元数据。
        cf := baseMetaFetcher.NewMetaFetcher(
            // 创建一个新的 BucketClient,用于与对象存储交互。
            extprom.WrapRegistererWithPrefix("thanos_", reg), []block.MetadataFilter{
                timePartitionMetaFilter,    // 创建一个新的 TimePartitionMetaFilter,用于过滤块元数据。
                labelShardedMetaFilter,     // 创建一个新的 LabelShardedMetaFilter,用于过滤块元数据。
                consistencyDelayMetaFilter, // 创建一个新的 ConsistencyDelayMetaFilter,用于过滤块元数据。
                ignoreDeletionMarkFilter,   // 创建一个新的 IgnoreDeletionMarkFilter,用于过滤块元数据。
                block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels), // 创建一个新的 ReplicaLabelRemover,用于过滤块元数据。
                duplicateBlocksFilter, // 创建一个新的 DuplicateBlocksFilter,用于过滤块元数据。
                noCompactMarkerFilter, // 创建一个新的 NoCompactMarkerFilter,用于过滤块元数据。
            },
        )
        // UpdateOnChange 更新块元数据。
        cf.UpdateOnChange(
            func(blocks []metadata.Meta, err error) /* 创建一个新的回调函数,用于处理块元数据。*/ {
                api.SetLoaded(blocks, err) // api.SetLoaded 设置已加载的块元数据。
            })
        // compact.NewMetaSyncer 创建一个新的 MetaSyncer,用于同步块元数据。
        sy, err = compact.NewMetaSyncer(
            logger,                   // 日志记录器。
            reg,                      // 注册器。
            bkt,                      // 对象存储客户端。
            cf,                       // 块元数据获取器。
            duplicateBlocksFilter,    // 块元数据过滤器。
            ignoreDeletionMarkFilter, // 块元数据过滤器。
            compactMetrics.blocksMarked.WithLabelValues(metadata.DeletionMarkFilename, ""), // 块元数据标记指标。
            compactMetrics.garbageCollectedBlocks,                                          // 垃圾收集块指标。
            conf.blockSyncConcurrency)                                                      // 块同步并发数。
        if err != nil {
            // 如果创建 MetaSyncer 失败,则返回错误。
            return errors.Wrap(err, "create syncer")
        }
    }
    // 调用compactions.levels方法,传入配置文件中的最大压缩级别conf.maxCompactionLevel,来创建一个新的压缩级别对象levels。如果在这个过程中出现错误,代码会返回一个错误,并使用errors.Wrap函数包装错误信息,以便于调试和错误追踪。
    levels, err := compactions.levels(conf.maxCompactionLevel)
    if err != nil {
        // 如果获取压缩级别失败,则返回错误。
        return errors.Wrap(err, "get compaction levels")
    }
 
    // compact.NewCompactor 创建一个新的 Compactor,用于处理块压缩。
    if conf.maxCompactionLevel < compactions.maxLevel() {
        //Max compaction level 是指允许的最大压缩级别。如果配置文件中的最大压缩级别conf.maxCompactionLevel小于compactions.maxLevel(),则记录一条警告日志,指出当前的最大压缩级别低于应该的级别。在这种情况下,为了提高性能和准确性,应该将最大压缩级别设置为默认值。因此,这里记录一条警告日志,以便于开发者或管理员注意到这个问题,并采取相应的措施进行修复。
        level.Warn(logger).Log("msg", "Max compaction level is lower than should be", "current", conf.maxCompactionLevel, "default", compactions.maxLevel())
    }
 
    ctx, cancel := context.WithCancel(context.Background())
    ctx = tracing.ContextWithTracer(ctx, tracer)
 
    defer func() {
        if rerr != nil {
            cancel()
        }
    }()
 
    var mergeFunc storage.VerticalChunkSeriesMergeFunc
    switch conf.dedupFunc {
    case compact.DedupAlgorithmPenalty:
        mergeFunc = dedup.NewChunkSeriesMerger()
 
        if len(conf.dedupReplicaLabels) == 0 {
            return errors.New("penalty based deduplication needs at least one replica label specified")
        }
    case "":
        mergeFunc = storage.NewCompactingChunkSeriesMerger(storage.ChainedSeriesMerge)
 
    default:
        return errors.Errorf("unsupported deduplication func, got %s", conf.dedupFunc)
    }
 
    // Instantiate the compactor with different time slices. Timestamps in TSDB
    // are in milliseconds.
    comp, err := tsdb.NewLeveledCompactor(ctx, reg, logger, levels, downsample.NewPool(), mergeFunc)
    if err != nil {
        return errors.Wrap(err, "create compactor")
    }
 
    var (
        compactDir      = path.Join(conf.dataDir, "compact")
        downsamplingDir = path.Join(conf.dataDir, "downsample")
    )
 
    if err := os.MkdirAll(compactDir, os.ModePerm); err != nil {
        return errors.Wrap(err, "create working compact directory")
    }
 
    if err := os.MkdirAll(downsamplingDir, os.ModePerm); err != nil {
        return errors.Wrap(err, "create working downsample directory")
    }
 
    grouper := compact.NewDefaultGrouper(
        logger,
        bkt,
        conf.acceptMalformedIndex,
        enableVerticalCompaction,
        reg,
        compactMetrics.blocksMarked.WithLabelValues(metadata.DeletionMarkFilename, ""),
        compactMetrics.garbageCollectedBlocks,
        compactMetrics.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.OutOfOrderChunksNoCompactReason),
        metadata.HashFunc(conf.hashFunc),
    )
    tsdbPlanner := compact.NewPlanner(logger, levels, noCompactMarkerFilter)
    planner := compact.WithLargeTotalIndexSizeFilter(
        tsdbPlanner,
        bkt,
        int64(conf.maxBlockIndexSize),
        compactMetrics.blocksMarked.WithLabelValues(metadata.NoCompactMarkFilename, metadata.IndexSizeExceedingNoCompactReason),
    )
    blocksCleaner := compact.NewBlocksCleaner(logger, bkt, ignoreDeletionMarkFilter, deleteDelay, compactMetrics.blocksCleaned, compactMetrics.blockCleanupFailures)
    compactor, err := compact.NewBucketCompactor(
        logger,
        sy,
        grouper,
        planner,
        comp,
        compactDir,
        bkt,
        conf.compactionConcurrency,
        conf.skipBlockWithOutOfOrderChunks,
    )
    if err != nil {
        return errors.Wrap(err, "create bucket compactor")
    }
 
    retentionByResolution := map[compact.ResolutionLevel]time.Duration{
        compact.ResolutionLevelRaw: time.Duration(conf.retentionRaw),
        compact.ResolutionLevel5m:  time.Duration(conf.retentionFiveMin),
        compact.ResolutionLevel1h:  time.Duration(conf.retentionOneHr),
    }
 
    if retentionByResolution[compact.ResolutionLevelRaw].Milliseconds() != 0 {
        // If downsampling is enabled, error if raw retention is not sufficient for downsampling to occur (upper bound 10 days for 1h resolution)
        if !conf.disableDownsampling && retentionByResolution[compact.ResolutionLevelRaw].Milliseconds() < downsample.ResLevel1DownsampleRange {
            return errors.New("raw resolution must be higher than the minimum block size after which 5m resolution downsampling will occur (40 hours)")
        }
        level.Info(logger).Log("msg", "retention policy of raw samples is enabled", "duration", retentionByResolution[compact.ResolutionLevelRaw])
    }
    if retentionByResolution[compact.ResolutionLevel5m].Milliseconds() != 0 {
        // If retention is lower than minimum downsample range, then no downsampling at this resolution will be persisted
        if !conf.disableDownsampling && retentionByResolution[compact.ResolutionLevel5m].Milliseconds() < downsample.ResLevel2DownsampleRange {
            return errors.New("5m resolution retention must be higher than the minimum block size after which 1h resolution downsampling will occur (10 days)")
        }
        level.Info(logger).Log("msg", "retention policy of 5 min aggregated samples is enabled", "duration", retentionByResolution[compact.ResolutionLevel5m])
    }
    if retentionByResolution[compact.ResolutionLevel1h].Milliseconds() != 0 {
        level.Info(logger).Log("msg", "retention policy of 1 hour aggregated samples is enabled", "duration", retentionByResolution[compact.ResolutionLevel1h])
    }
 
    var cleanMtx sync.Mutex
    // TODO(GiedriusS): we could also apply retention policies here but the logic would be a bit more complex.
    cleanPartialMarked := func() error {
        cleanMtx.Lock()
        defer cleanMtx.Unlock()
 
        if err := sy.SyncMetas(ctx); err != nil {
            cancel()
            return errors.Wrap(err, "syncing metas")
        }
 
        compact.BestEffortCleanAbortedPartialUploads(ctx, logger, sy.Partial(), bkt, compactMetrics.partialUploadDeleteAttempts, compactMetrics.blocksCleaned, compactMetrics.blockCleanupFailures)
        if err := blocksCleaner.DeleteMarkedBlocks(ctx); err != nil {
            return errors.Wrap(err, "cleaning marked blocks")
        }
        compactMetrics.cleanups.Inc()
 
        return nil
    }
 
    compactMainFn := func() error {
        if err := compactor.Compact(ctx); err != nil {
            return errors.Wrap(err, "compaction")
        }
 
        if !conf.disableDownsampling {
            // After all compactions are done, work down the downsampling backlog.
            // We run two passes of this to ensure that the 1h downsampling is generated
            // for 5m downsamplings created in the first run.
            level.Info(logger).Log("msg", "start first pass of downsampling")
            if err := sy.SyncMetas(ctx); err != nil {
                return errors.Wrap(err, "sync before first pass of downsampling")
            }
 
            for _, meta := range sy.Metas() {
                groupKey := meta.Thanos.GroupKey()
                downsampleMetrics.downsamples.WithLabelValues(groupKey)
                downsampleMetrics.downsampleFailures.WithLabelValues(groupKey)
            }
            if err := downsampleBucket(ctx, logger, downsampleMetrics, bkt, sy.Metas(), downsamplingDir, conf.downsampleConcurrency, metadata.HashFunc(conf.hashFunc)); err != nil {
                return errors.Wrap(err, "first pass of downsampling failed")
            }
 
            level.Info(logger).Log("msg", "start second pass of downsampling")
            if err := sy.SyncMetas(ctx); err != nil {
                return errors.Wrap(err, "sync before second pass of downsampling")
            }
            if err := downsampleBucket(ctx, logger, downsampleMetrics, bkt, sy.Metas(), downsamplingDir, conf.downsampleConcurrency, metadata.HashFunc(conf.hashFunc)); err != nil {
                return errors.Wrap(err, "second pass of downsampling failed")
            }
            level.Info(logger).Log("msg", "downsampling iterations done")
        } else {
            level.Info(logger).Log("msg", "downsampling was explicitly disabled")
        }
 
        // TODO(bwplotka): Find a way to avoid syncing if no op was done.
        if err := sy.SyncMetas(ctx); err != nil {
            return errors.Wrap(err, "sync before retention")
        }
 
        if err := compact.ApplyRetentionPolicyByResolution(ctx, logger, bkt, sy.Metas(), retentionByResolution, compactMetrics.blocksMarked.WithLabelValues(metadata.DeletionMarkFilename, "")); err != nil {
            return errors.Wrap(err, "retention failed")
        }
 
        return cleanPartialMarked()
    }
    // 启动并发任务: 使用 g.Add 启动一个新的并发任务,该任务在 ctx 上下文中运行。
    g.Add(func() error {
        defer runutil.CloseWithLogOnErr(logger, bkt, "bucket client")
 
        if !conf.wait {
            return compactMainFn()
        }
 
        // --wait=true is specified.
        // runutil.Repeat 是一个无限循环,它会每隔一段时间执行一次给定的函数。如果函数返回一个错误,它会等待一段时间后再次尝试。
        return runutil.Repeat(conf.waitInterval, ctx.Done(), func() error {
            err := compactMainFn()
            if err == nil {
                compactMetrics.iterations.Inc()
                return nil
            }
 
            // The HaltError type signals that we hit a critical bug and should block
            // for investigation. You should alert on this being halted.
            // compact.IsHaltError 是一个函数,用于检查给定的错误是否表示一个关键的、需要调查的错误。如果返回 true,则表明这是一个关键错误。如果返回 false,则表明这是一个普通的错误。
            if compact.IsHaltError(err) {
                if conf.haltOnError {
                    level.Error(logger).Log("msg", "critical error detected; halting", "err", err)
                    compactMetrics.halted.Set(1)
                    select {}
                } else {
                    return errors.Wrap(err, "critical error detected")
                }
            }
 
            // The RetryError signals that we hit an retriable error (transient error, no connection).
            // You should alert on this being triggered too frequently.
            if compact.IsRetryError(err) {
                level.Error(logger).Log("msg", "retriable error", "err", err)
                compactMetrics.retried.Inc()
                // TODO(bplotka): use actual "retry()" here instead of waiting 5 minutes?
                return nil
            }
 
            return errors.Wrap(err, "error executing compaction")
        })
    }, func(error) {
        cancel()
    })
 
    if conf.wait {
        r := route.New()
 
        ins := extpromhttp.NewInstrumentationMiddleware(reg, nil)
 
        global := ui.NewBucketUI(logger, conf.webConf.externalPrefix, conf.webConf.prefixHeaderName, component)
        global.Register(r, ins)
 
        // Configure Request Logging for HTTP calls.
        opts := []logging.Option{logging.WithDecider(func(_ string, _ error) logging.Decision {
            return logging.NoLogCall
        })}
        logMiddleware := logging.NewHTTPServerMiddleware(logger, opts...)
        api.Register(r.WithPrefix("/api/v1"), tracer, logger, ins, logMiddleware)
 
        // Separate fetcher for global view.
        // TODO(bwplotka): Allow Bucket UI to visualize the state of the block as well.
        f := baseMetaFetcher.NewMetaFetcher(extprom.WrapRegistererWithPrefix("thanos_bucket_ui", reg), nil, "component", "globalBucketUI")
        f.UpdateOnChange(func(blocks []metadata.Meta, err error) {
            api.SetGlobal(blocks, err)
        })
 
        srv.Handle("/", r)
 
        // Periodically remove partial blocks and blocks marked for deletion
        // since one iteration potentially could take a long time.
        if conf.cleanupBlocksInterval > 0 {
            g.Add(func() error {
                return runutil.Repeat(conf.cleanupBlocksInterval, ctx.Done(), cleanPartialMarked)
            }, func(error) {
                cancel()
            })
        }
 
        // Periodically calculate the progress of compaction, downsampling and retention.
        if conf.progressCalculateInterval > 0 {
            g.Add(func() error {
                ps := compact.NewCompactionProgressCalculator(reg, tsdbPlanner)
                rs := compact.NewRetentionProgressCalculator(reg, retentionByResolution)
                var ds *compact.DownsampleProgressCalculator
                if !conf.disableDownsampling {
                    ds = compact.NewDownsampleProgressCalculator(reg)
                }
 
                return runutil.Repeat(conf.progressCalculateInterval, ctx.Done(), func() error {
 
                    if err := sy.SyncMetas(ctx); err != nil {
                        return errors.Wrapf(err, "could not sync metas")
                    }
 
                    metas := sy.Metas()
                    groups, err := grouper.Groups(metas)
                    if err != nil {
                        return errors.Wrapf(err, "could not group metadata for compaction")
                    }
 
                    if err = ps.ProgressCalculate(ctx, groups); err != nil {
                        return errors.Wrapf(err, "could not calculate compaction progress")
                    }
 
                    retGroups, err := grouper.Groups(metas)
                    if err != nil {
                        return errors.Wrapf(err, "could not group metadata for retention")
                    }
 
                    if err = rs.ProgressCalculate(ctx, retGroups); err != nil {
                        return errors.Wrapf(err, "could not calculate retention progress")
                    }
 
                    if !conf.disableDownsampling {
                        groups, err = grouper.Groups(metas)
                        if err != nil {
                            return errors.Wrapf(err, "could not group metadata into downsample groups")
                        }
                        if err := ds.ProgressCalculate(ctx, groups); err != nil {
                            return errors.Wrapf(err, "could not calculate downsampling progress")
                        }
                    }
 
                    return nil
                })
            }, func(err error) {
                cancel()
            })
        }
 
        /*
            g.Add(func() error { ... }, func(error) { ... }):
                向一个goroutine管理器g中添加一个任务。这个管理器用于并发地执行任务,并在所有任务完成后进行清理或其他操作。这个任务有两个部分:一个是执行的任务本身(返回error),另一个是错误处理函数。
                在第一个部分,我们看到它尝试同步元数据(为了获取最新的块信息),然后根据这些信息进行分组,并计算压缩进度、保留进度和下采样进度。如果在这个过程中出现错误,它会返回这个错误。
        */
        g.Add(func() error {
            // 这行代码创建了一个新的上下文iterCtx,它有一个超时时间conf.blockViewerSyncBlockTimeout。这个上下文用于控制f.Fetch(iterCtx)操作的超时。iterCancel是一个取消函数,用于在必要时取消这个上下文。
            iterCtx, iterCancel := context.WithTimeout(ctx, conf.blockViewerSyncBlockTimeout)
            // 调用iterCancel()取消iterCtx,这是因为在单次数据获取完成后,不需要保持这个上下文活跃。
            _, _, _ = f.Fetch(iterCtx)
            iterCancel()
 
            // For /global state make sure to fetch periodically.
            // 使用runutil.Repeat函数定期执行任务。它每隔conf.blockViewerSyncBlockInterval时间执行一次给定的函数,直到ctx.Done()通道被关闭(表示上下文ctx被取消或超时)。
            return runutil.Repeat(conf.blockViewerSyncBlockInterval, ctx.Done(), func() error {
                // 在runutil.Repeat的回调函数中,使用了runutil.RetryWithLog来尝试执行一个操作,如果失败则重试。这里,它最多等待time.Minute来等待操作成功,如果在ctx.Done()之前成功,则停止重试。
                return runutil.RetryWithLog(logger, time.Minute, ctx.Done(), func() error {
                    // 这行代码再次创建了一个带有超时的上下文,用于f.Fetch(iterCtx)操作的这次尝试。iterCancel是一个取消函数,用于在必要时取消这个上下文。
                    iterCtx, iterCancel := context.WithTimeout(ctx, conf.blockViewerSyncBlockTimeout)
                    // 使用defer关键字确保在函数返回前调用iterCancel(),无论是因为成功返回还是因为遇到错误返回。
                    defer iterCancel()
 
                    // 尝试从数据源获取数据,这次错误err被捕获并返回,以便runutil.RetryWithLog可以根据这个错误决定是否重试。
                    _, _, err := f.Fetch(iterCtx)
                    return err
                })
            })
        }, func(error) { // 如果主任务函数返回错误,这个错误处理函数会被调用。这里,它调用cancel()函数,可能是用来取消整个操作或通知其他等待这个操作的goroutine。
            cancel()
        })
    }
 
    level.Info(logger).Log("msg", "starting compact node") // 记录日志,表明正在启动压缩节点
    statusProber.Ready()                                   // 标记为就绪状态
    return nil
}
 
type compactConfig struct {
    haltOnError                                    bool                  // 关键压缩错误检测时是否停止进程。默认是true。
    acceptMalformedIndex                           bool                  // 是否接受损坏的索引文件。默认是false。
    maxCompactionLevel                             int                   // 最大压缩级别。默认是2。
    http                                           httpConfig            // HTTP配置
    dataDir                                        string                // 数据目录
    objStore                                       extflag.PathOrContent // 对象存储配置
    consistencyDelay                               time.Duration         // 延迟一致性检查的时间。默认是0。
    retentionRaw, retentionFiveMin, retentionOneHr model.Duration        // 原始数据、5分钟数据、1小时数据的保留时间
    wait                                           bool                  // 是否等待压缩完成。默认是false。
    waitInterval                                   time.Duration         // 等待压缩完成的时间间隔。默认是1分钟。
    disableDownsampling                            bool                  // 是否禁用下采样。默认是false。
    blockSyncConcurrency                           int                   // 同步块时的并发数。默认是20。
    blockMetaFetchConcurrency                      int                   // 获取块元数据时的并发数。默认是20。
    blockViewerSyncBlockInterval                   time.Duration         // 同步块的时间间隔。默认是10分钟。
    blockViewerSyncBlockTimeout                    time.Duration         // 同步块的超时时间。默认是1分钟。
    cleanupBlocksInterval                          time.Duration         // 清理块的时间间隔。默认是1小时。
    compactionConcurrency                          int                   // 压缩时的并发数。默认是1。
    downsampleConcurrency                          int                   // 下采样时的并发数。默认是1。
    deleteDelay                                    model.Duration        // 删除块的时间延迟。默认是1小时。
    dedupReplicaLabels                             []string              // 需要删除的副本标签。默认是["replica"]。
    selectorRelabelConf                            extflag.PathOrContent // 选择器重新标签配置。默认是空字符串。
    webConf                                        webConfig             // web配置。默认是空字符串。
    label                                          string                // 标签。默认是空字符串。
    maxBlockIndexSize                              units.Base2Bytes      // 最大块索引大小。默认是512MB。
    hashFunc                                       string                // 哈希函数。默认是fnv1a。
    enableVerticalCompaction                       bool                  // 启用垂直压缩。默认是true。
    dedupFunc                                      string                // 去重函数。默认是none。
    skipBlockWithOutOfOrderChunks                  bool                  // 跳过包含乱序块的块。默认是true。
    progressCalculateInterval                      time.Duration         // 进度计算间隔。默认是5分钟。
    filterConf                                     *store.FilterConfig   // 过滤器配置。默认是空字符串。
}
 
// registerFlag 方法用于注册各种配置标志到命令行参数中。
// 它主要用于配置Thanos Compactor组件的行为。
func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) {
    // 注册debug.halt-on-error标志
    cmd.Flag("debug.halt-on-error", "Halt the process if a critical compaction error is detected.").
        Hidden().Default("true").BoolVar(&cc.haltOnError)
 
    // 注册debug.accept-malformed-index标志
    cmd.Flag("debug.accept-malformed-index",
        "Compaction index verification will ignore out of order label names.").
        Hidden().Default("false").BoolVar(&cc.acceptMalformedIndex)
 
    // 注册debug.max-compaction-level标志
    cmd.Flag("debug.max-compaction-level", fmt.Sprintf("Maximum compaction level, default is %d: %s", compactions.maxLevel(), compactions.String())).
        Hidden().Default(strconv.Itoa(compactions.maxLevel())).IntVar(&cc.maxCompactionLevel)
 
    // 注册HTTP相关标志
    cc.http.registerFlag(cmd)
 
    // 注册data-dir标志
    cmd.Flag("data-dir", "Data directory in which to cache blocks and process compactions.").
        Default("./data").StringVar(&cc.dataDir)
 
    // 注册对象存储标志
    cc.objStore = *extkingpin.RegisterCommonObjStoreFlags(cmd, "", false)
 
    // 注册consistency-delay标志
    cmd.Flag("consistency-delay", fmt.Sprintf("Minimum age of fresh (non-compacted) blocks before they are being processed. Malformed blocks older than the maximum of consistency-delay and %v will be removed.", compact.PartialUploadThresholdAge)).
        Default("30m").DurationVar(&cc.consistencyDelay)
 
    // 注册retention相关标志
    cmd.Flag("retention.resolution-raw",
        "How long to retain raw samples in bucket. Setting this to 0d will retain samples of this resolution forever").
        Default("0d").SetValue(&cc.retentionRaw)
    // 注册retention.resolution-5m和retention.resolution-1h标志
    cmd.Flag("retention.resolution-5m", "How long to retain samples of resolution 1 (5 minutes) in bucket. Setting this to 0d will retain samples of this resolution forever").
        Default("0d").SetValue(&cc.retentionFiveMin)
    // 注册retention.resolution-1h标志
    cmd.Flag("retention.resolution-1h", "How long to retain samples of resolution 2 (1 hour) in bucket. Setting this to 0d will retain samples of this resolution forever").
        Default("0d").SetValue(&cc.retentionOneHr)
 
    // 注册wait标志
    // TODO(kakkoyun, pgough): https://github.com/thanos-io/thanos/issues/2266.
    cmd.Flag("wait", "Do not exit after all compactions have been processed and wait for new work.").
        Short('w').BoolVar(&cc.wait)
 
    // 注册wait-interval标志
    cmd.Flag("wait-interval", "Wait interval between consecutive compaction runs and bucket refreshes. Only works when --wait flag specified.").
        Default("5m").DurationVar(&cc.waitInterval)
 
    // 注册downsampling相关标志
    cmd.Flag("downsampling.disable", "Disables downsampling. This is not recommended "+
        "as querying long time ranges without non-downsampled data is not efficient and useful e.g it is not possible to render all samples for a human eye anyway").
        Default("false").BoolVar(&cc.disableDownsampling)
 
    // 注册block-sync-concurrency标志
    cmd.Flag("block-sync-concurrency", "Number of goroutines to use when syncing block metadata from object storage.").
        Default("20").IntVar(&cc.blockSyncConcurrency)
 
    // 注册block-meta-fetch-concurrency标志
    cmd.Flag("block-meta-fetch-concurrency", "Number of goroutines to use when fetching block metadata from object storage.").
        Default("32").IntVar(&cc.blockMetaFetchConcurrency)
 
    // 注册block-viewer.global.sync-block-interval标志
    cmd.Flag("block-viewer.global.sync-block-interval", "Repeat interval for syncing the blocks between local and remote view for /global Block Viewer UI.").
        Default("1m").DurationVar(&cc.blockViewerSyncBlockInterval)
 
    // 注册block-viewer.global.sync-block-timeout标志
    cmd.Flag("block-viewer.global.sync-block-timeout", "Maximum time for syncing the blocks between local and remote view for /global Block Viewer UI.").
        Default("5m").DurationVar(&cc.blockViewerSyncBlockTimeout)
 
    // 注册compact.cleanup-interval标志
    cmd.Flag("compact.cleanup-interval", "How often we should clean up partially uploaded blocks and blocks with deletion mark in the background when --wait has been enabled. Setting it to \"0s\" disables it - the cleaning will only happen at the end of an iteration.").
        Default("5m").DurationVar(&cc.cleanupBlocksInterval)
 
    // 注册compact.progress-interval标志
    cmd.Flag("compact.progress-interval", "Frequency of calculating the compaction progress in the background when --wait has been enabled. Setting it to \"0s\" disables it. Now compaction, downsampling and retention progress are supported.").
        Default("5m").DurationVar(&cc.progressCalculateInterval)
 
    // 注册compact.concurrency标志
    cmd.Flag("compact.concurrency", "Number of goroutines to use when compacting groups.").
        Default("1").IntVar(&cc.compactionConcurrency)
 
    // 注册downsample.concurrency标志
    cmd.Flag("downsample.concurrency", "Number of goroutines to use when downsampling blocks.").
        Default("1").IntVar(&cc.downsampleConcurrency)
 
    // 注册delete-delay标志
    cmd.Flag("delete-delay", "Time before a block marked for deletion is deleted from bucket. "+
        "If delete-delay is non zero, blocks will be marked for deletion and compactor component will delete blocks marked for deletion from the bucket. "+
        "If delete-delay is 0, blocks will be deleted straight away. "+
        "Note that deleting blocks immediately can cause query failures, if store gateway still has the block loaded, "+
        "or compactor is ignoring the deletion because it's compacting the block at the same time.").
        Default("48h").SetValue(&cc.deleteDelay)
 
    // 注册compact.enable-vertical-compaction标志
    cmd.Flag("compact.enable-vertical-compaction", "Experimental. When set to true, compactor will allow overlaps and perform **irreversible** vertical compaction. See https://thanos.io/tip/components/compact.md/#vertical-compactions to read more. "+
        "Please note that by default this uses a NAIVE algorithm for merging. If you need a different deduplication algorithm (e.g one that works well with Prometheus replicas), please set it via --deduplication.func."+
        "NOTE: This flag is ignored and (enabled) when --deduplication.replica-label flag is set.").
        Hidden().Default("false").BoolVar(&cc.enableVerticalCompaction)
 
    // 注册deduplication.func标志
    cmd.Flag("deduplication.func", "Experimental. Deduplication algorithm for merging overlapping blocks. "+
        "Possible values are: \"\", \"penalty\". If no value is specified, the default compact deduplication merger is used, which performs 1:1 deduplication for samples. "+
        "When set to penalty, penalty based deduplication algorithm will be used. At least one replica label has to be set via --deduplication.replica-label flag.").
        Default("").EnumVar(&cc.dedupFunc, compact.DedupAlgorithmPenalty, "")
 
    // 注册deduplication.replica-label标志
    cmd.Flag("deduplication.replica-label", "Label to treat as a replica indicator of blocks that can be deduplicated (repeated flag). This will merge multiple replica blocks into one. This process is irreversible."+
        "Experimental. When one or more labels are set, compactor will ignore the given labels so that vertical compaction can merge the blocks."+
        "Please note that by default this uses a NAIVE algorithm for merging which works well for deduplication of blocks with **precisely the same samples** like produced by Receiver replication."+
        "If you need a different deduplication algorithm (e.g one that works well with Prometheus replicas), please set it via --deduplication.func.").
        StringsVar(&cc.dedupReplicaLabels)
 
    // 注册compact.block-max-index-size标志
    // TODO(bwplotka): This is short term fix for https://github.com/thanos-io/thanos/issues/1424, replace with vertical block sharding https://github.com/thanos-io/thanos/pull/3390.
    cmd.Flag("compact.block-max-index-size", "Maximum index size for the resulted block during any compaction. Note that"+
        "total size is approximated in worst case. If the block that would be resulted from compaction is estimated to exceed this number, biggest source"+
        "block is marked for no compaction (no-compact-mark.json is uploaded) which causes this block to be excluded from any compaction. "+
        "Default is due to https://github.com/thanos-io/thanos/issues/1424, but it's overall recommended to keeps block size to some reasonable size.").
        Hidden().Default("64GB").BytesVar(&cc.maxBlockIndexSize)
 
    // 注册compact.skip-block-with-out-of-order-chunks标志
    cmd.Flag("compact.skip-block-with-out-of-order-chunks", "When set to true, mark blocks containing index with out-of-order chunks for no compact instead of halting the compaction").
        Hidden().Default("false").BoolVar(&cc.skipBlockWithOutOfOrderChunks)
 
    // 注册hash-func标志
    cmd.Flag("hash-func", "Specify which hash function to use when calculating the hashes of produced files. If no function has been specified, it does not happen. This permits avoiding downloading some files twice albeit at some performance cost. Possible values are: \"\", \"SHA256\".").
        Default("").EnumVar(&cc.hashFunc, "SHA256", "")
 
    // 初始化filter配置
    cc.filterConf = &store.FilterConfig{}
 
    // 注册min-time标志
    cmd.Flag("min-time", "Start of time range limit to compact. Thanos Compactor will compact only blocks, which happened later than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y.").
        Default("0000-01-01T00:00:00Z").SetValue(&cc.filterConf.MinTime)
 
    // 注册max-time标志
    cmd.Flag("max-time", "End of time range limit to compact. Thanos Compactor will compact only blocks, which happened earlier than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y.").
        Default("9999-12-31T23:59:59Z").SetValue(&cc.filterConf.MaxTime)
 
    // 注册selector-relabel相关标志
    cc.selectorRelabelConf = *extkingpin.RegisterSelectorRelabelFlags(cmd)
 
    // 注册web相关标志
    cc.webConf.registerFlag(cmd)
 
    // 注册bucket-web-label标志
    cmd.Flag("bucket-web-label", "Prometheus label to use as timeline title in the bucket web UI").StringVar(&cc.label)
}
posted @   左扬  阅读(6)  评论(0编辑  收藏  举报
相关博文:
阅读排行:
· 在鹅厂做java开发是什么体验
· 百万级群聊的设计实践
· WPF到Web的无缝过渡:英雄联盟客户端的OpenSilver迁移实战
· 永远不要相信用户的输入:从 SQL 注入攻防看输入验证的重要性
· 浏览器原生「磁吸」效果!Anchor Positioning 锚点定位神器解析
历史上的今天:
2022-02-07 Go从入门到精通——指针
2022-02-07 Go从入门到精通——搭建Go开发环境
2022-02-07 Go从入门到精通——如何安装 Go 语言开发包?
2021-02-07 RocketMQ(4.8.0)——生产者
2021-02-07 RocketMQ概述
levels of contents
点击右上角即可分享
微信分享提示