Support V210 train and fix some bugs and improve

2023-12-23 12:59:57 +09:00
parent 3090f5837d
commit 6c171fcea0
8 changed files with 1365 additions and 900 deletions
--- a/data_utils.py
+++ b/data_utils.py
@@ -299,6 +299,10 @@ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
        self.boundaries = boundaries

        self.buckets, self.num_samples_per_bucket = self._create_buckets()
+        logger.info(f"Bucket info: {self.num_samples_per_bucket}")
+        logger.info(
+            f"Unuseful samples: {len(self.lengths) - sum(self.num_samples_per_bucket)}"
+        )
        self.total_size = sum(self.num_samples_per_bucket)
        self.num_samples = self.total_size // self.num_replicas