| --- |
| language: |
| - en |
| license: apache-2.0 |
| tags: |
| - sentence-transformers |
| - sparse-encoder |
| - sparse |
| - splade |
| - generated_from_trainer |
| - dataset_size:99000 |
| - loss:SpladeLoss |
| - loss:SparseMultipleNegativesRankingLoss |
| - loss:FlopsLoss |
| base_model: distilbert/distilbert-base-uncased |
| widget: |
| - text: How do I know if a girl likes me at school? |
| - text: What are some five star hotel in Jaipur? |
| - text: Is it normal to fantasize your wife having sex with another man? |
| - text: What is the Sahara, and how do the average temperatures there compare to the |
| ones in the Simpson Desert? |
| - text: What are Hillary Clinton's most recognized accomplishments while Secretary |
| of State? |
| datasets: |
| - sentence-transformers/quora-duplicates |
| pipeline_tag: feature-extraction |
| library_name: sentence-transformers |
| metrics: |
| - cosine_accuracy |
| - cosine_accuracy_threshold |
| - cosine_f1 |
| - cosine_f1_threshold |
| - cosine_precision |
| - cosine_recall |
| - cosine_ap |
| - cosine_mcc |
| - dot_accuracy |
| - dot_accuracy_threshold |
| - dot_f1 |
| - dot_f1_threshold |
| - dot_precision |
| - dot_recall |
| - dot_ap |
| - dot_mcc |
| - euclidean_accuracy |
| - euclidean_accuracy_threshold |
| - euclidean_f1 |
| - euclidean_f1_threshold |
| - euclidean_precision |
| - euclidean_recall |
| - euclidean_ap |
| - euclidean_mcc |
| - manhattan_accuracy |
| - manhattan_accuracy_threshold |
| - manhattan_f1 |
| - manhattan_f1_threshold |
| - manhattan_precision |
| - manhattan_recall |
| - manhattan_ap |
| - manhattan_mcc |
| - max_accuracy |
| - max_accuracy_threshold |
| - max_f1 |
| - max_f1_threshold |
| - max_precision |
| - max_recall |
| - max_ap |
| - max_mcc |
| - active_dims |
| - sparsity_ratio |
| - dot_accuracy@1 |
| - dot_accuracy@3 |
| - dot_accuracy@5 |
| - dot_accuracy@10 |
| - dot_precision@1 |
| - dot_precision@3 |
| - dot_precision@5 |
| - dot_precision@10 |
| - dot_recall@1 |
| - dot_recall@3 |
| - dot_recall@5 |
| - dot_recall@10 |
| - dot_ndcg@10 |
| - dot_mrr@10 |
| - dot_map@100 |
| - query_active_dims |
| - query_sparsity_ratio |
| - corpus_active_dims |
| - corpus_sparsity_ratio |
| co2_eq_emissions: |
| emissions: 1.4164940270091377 |
| energy_consumed: 0.02527693261851813 |
| source: codecarbon |
| training_type: fine-tuning |
| on_cloud: false |
| cpu_model: AMD Ryzen 9 6900HX with Radeon Graphics |
| ram_total_size: 30.6114501953125 |
| hours_used: 0.222 |
| hardware_used: 1 x NVIDIA GeForce RTX 3070 Ti Laptop GPU |
| model-index: |
| - name: splade-distilbert-base-uncased trained on Quora Duplicates Questions |
| results: |
| - task: |
| type: sparse-binary-classification |
| name: Sparse Binary Classification |
| dataset: |
| name: quora duplicates dev |
| type: quora_duplicates_dev |
| metrics: |
| - type: cosine_accuracy |
| value: 0.758 |
| name: Cosine Accuracy |
| - type: cosine_accuracy_threshold |
| value: 0.8166326284408569 |
| name: Cosine Accuracy Threshold |
| - type: cosine_f1 |
| value: 0.6792899408284023 |
| name: Cosine F1 |
| - type: cosine_f1_threshold |
| value: 0.5695896148681641 |
| name: Cosine F1 Threshold |
| - type: cosine_precision |
| value: 0.5487571701720841 |
| name: Cosine Precision |
| - type: cosine_recall |
| value: 0.8913043478260869 |
| name: Cosine Recall |
| - type: cosine_ap |
| value: 0.6887627674706448 |
| name: Cosine Ap |
| - type: cosine_mcc |
| value: 0.508171027288805 |
| name: Cosine Mcc |
| - type: dot_accuracy |
| value: 0.765 |
| name: Dot Accuracy |
| - type: dot_accuracy_threshold |
| value: 51.6699104309082 |
| name: Dot Accuracy Threshold |
| - type: dot_f1 |
| value: 0.6762028608582575 |
| name: Dot F1 |
| - type: dot_f1_threshold |
| value: 46.524925231933594 |
| name: Dot F1 Threshold |
| - type: dot_precision |
| value: 0.5816554809843401 |
| name: Dot Precision |
| - type: dot_recall |
| value: 0.8074534161490683 |
| name: Dot Recall |
| - type: dot_ap |
| value: 0.6335823489360819 |
| name: Dot Ap |
| - type: dot_mcc |
| value: 0.4996270089694481 |
| name: Dot Mcc |
| - type: euclidean_accuracy |
| value: 0.677 |
| name: Euclidean Accuracy |
| - type: euclidean_accuracy_threshold |
| value: -14.272356986999512 |
| name: Euclidean Accuracy Threshold |
| - type: euclidean_f1 |
| value: 0.48599545798637395 |
| name: Euclidean F1 |
| - type: euclidean_f1_threshold |
| value: -0.6444530487060547 |
| name: Euclidean F1 Threshold |
| - type: euclidean_precision |
| value: 0.3213213213213213 |
| name: Euclidean Precision |
| - type: euclidean_recall |
| value: 0.9968944099378882 |
| name: Euclidean Recall |
| - type: euclidean_ap |
| value: 0.2032823056922341 |
| name: Euclidean Ap |
| - type: euclidean_mcc |
| value: -0.04590966956831287 |
| name: Euclidean Mcc |
| - type: manhattan_accuracy |
| value: 0.677 |
| name: Manhattan Accuracy |
| - type: manhattan_accuracy_threshold |
| value: -161.77682495117188 |
| name: Manhattan Accuracy Threshold |
| - type: manhattan_f1 |
| value: 0.48599545798637395 |
| name: Manhattan F1 |
| - type: manhattan_f1_threshold |
| value: -3.0494537353515625 |
| name: Manhattan F1 Threshold |
| - type: manhattan_precision |
| value: 0.3213213213213213 |
| name: Manhattan Precision |
| - type: manhattan_recall |
| value: 0.9968944099378882 |
| name: Manhattan Recall |
| - type: manhattan_ap |
| value: 0.20444314945561334 |
| name: Manhattan Ap |
| - type: manhattan_mcc |
| value: -0.04590966956831287 |
| name: Manhattan Mcc |
| - type: max_accuracy |
| value: 0.765 |
| name: Max Accuracy |
| - type: max_accuracy_threshold |
| value: 51.6699104309082 |
| name: Max Accuracy Threshold |
| - type: max_f1 |
| value: 0.6792899408284023 |
| name: Max F1 |
| - type: max_f1_threshold |
| value: 46.524925231933594 |
| name: Max F1 Threshold |
| - type: max_precision |
| value: 0.5816554809843401 |
| name: Max Precision |
| - type: max_recall |
| value: 0.9968944099378882 |
| name: Max Recall |
| - type: max_ap |
| value: 0.6887627674706448 |
| name: Max Ap |
| - type: max_mcc |
| value: 0.508171027288805 |
| name: Max Mcc |
| - type: active_dims |
| value: 78.32280731201172 |
| name: Active Dims |
| - type: sparsity_ratio |
| value: 0.9974338900690646 |
| name: Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoMSMARCO |
| type: NanoMSMARCO |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.22 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.42 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.52 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.76 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.22 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.13999999999999999 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.10400000000000001 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.07600000000000001 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.22 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.42 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.52 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.76 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.45321847177875746 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.3601269841269841 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.37334906504034243 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 74.76000213623047 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9975506191554868 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 103.06523895263672 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9966232475279261 |
| name: Corpus Sparsity Ratio |
| - type: dot_accuracy@1 |
| value: 0.22 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.42 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.52 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.76 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.22 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.13999999999999999 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.10400000000000001 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.07600000000000001 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.22 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.42 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.52 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.76 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.45321847177875746 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.3601269841269841 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.37334906504034243 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 74.76000213623047 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9975506191554868 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 103.06523895263672 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9966232475279261 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoNQ |
| type: NanoNQ |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.38 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.54 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.62 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.62 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.38 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.18 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.12400000000000003 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.06400000000000002 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.36 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.52 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.6 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.61 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.4828377104499333 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.4536666666666666 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.445384784044708 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 74.73999786376953 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9975512745605213 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 141.31478881835938 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9953700678586476 |
| name: Corpus Sparsity Ratio |
| - type: dot_accuracy@1 |
| value: 0.38 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.54 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.62 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.62 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.38 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.18 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.12400000000000003 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.06400000000000002 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.36 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.52 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.6 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.61 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.4828377104499333 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.4536666666666666 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.445384784044708 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 74.73999786376953 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9975512745605213 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 141.31478881835938 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9953700678586476 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoNFCorpus |
| type: NanoNFCorpus |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.34 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.5 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.54 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.58 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.34 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.30666666666666664 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.26 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.198 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.011597172822497613 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.06058581579610722 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.08260772201759854 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.09800124609193644 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.2466972614666078 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.42200000000000004 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.09401937795309984 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 79.69999694824219 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9973887688569477 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 202.17269897460938 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9933761647672298 |
| name: Corpus Sparsity Ratio |
| - type: dot_accuracy@1 |
| value: 0.34 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.5 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.54 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.58 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.34 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.30666666666666664 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.26 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.198 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.011597172822497613 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.06058581579610722 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.08260772201759854 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.09800124609193644 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.2466972614666078 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.42200000000000004 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.09401937795309984 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 79.69999694824219 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9973887688569477 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 202.17269897460938 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9933761647672298 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoQuoraRetrieval |
| type: NanoQuoraRetrieval |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.94 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.98 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.98 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.98 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.94 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.3933333333333333 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.24799999999999997 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.13199999999999998 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.8173333333333332 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.9279999999999999 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.946 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.97 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.9467235239993945 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.96 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.9290737327188939 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 76.58000183105469 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9974909900455063 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 77.59056854248047 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9974578805929336 |
| name: Corpus Sparsity Ratio |
| - type: dot_accuracy@1 |
| value: 0.94 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.98 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.98 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.98 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.94 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.3933333333333333 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.24799999999999997 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.13199999999999998 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.8173333333333332 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.9279999999999999 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.946 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.97 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.9467235239993945 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.96 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.9290737327188939 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 76.58000183105469 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9974909900455063 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 77.59056854248047 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9974578805929336 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-nano-beir |
| name: Sparse Nano BEIR |
| dataset: |
| name: NanoBEIR mean |
| type: NanoBEIR_mean |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.47 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.61 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.665 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.735 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.47 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.255 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.184 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.1175 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.3522326265389577 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.4821464539490268 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.5371519305043997 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.6095003115229841 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.5323692419236733 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.5489484126984127 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.46045673993926106 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 76.44499969482422 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9974954131546155 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 122.79780664247188 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9959767444255792 |
| name: Corpus Sparsity Ratio |
| - type: dot_accuracy@1 |
| value: 0.4359811616954475 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.6088540031397174 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.6659026687598116 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.7383987441130299 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.4359811616954475 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.2725170068027211 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.2089481946624804 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.14605965463108322 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.2532746332292894 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.3813452238818861 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.4363867898661836 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.5099503000039356 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.4684519639817077 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.5328029827315542 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.39738635557561647 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 90.39137197532713 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9970384846348428 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 152.36685474307478 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9950079662295042 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoClimateFEVER |
| type: NanoClimateFEVER |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.18 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.32 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.4 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.48 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.18 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.10666666666666666 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.08400000000000002 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.054000000000000006 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.085 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.14666666666666667 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.17833333333333332 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.215 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.1845115403570178 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.2674126984126984 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.1475834110231865 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 89.86000061035156 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9970558940891701 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 221.75527954101562 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.992734575730915 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoDBPedia |
| type: NanoDBPedia |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.6 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.84 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.84 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.92 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.6 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.5266666666666666 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.456 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.4220000000000001 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.04570544957623723 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.15367137863132574 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.1908008582920462 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.293554014064817 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.5070720730882787 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.7147222222222225 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.3906658166774757 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 69.5199966430664 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.997722298779796 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 135.93350219726562 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9955463763122578 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoFEVER |
| type: NanoFEVER |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.58 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.76 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.8 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.86 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.58 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.26666666666666666 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.16799999999999998 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.09 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.5466666666666666 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.7466666666666667 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.7866666666666667 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.8466666666666667 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.7069849294263234 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.6765000000000001 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.6651380090497737 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 89.87999725341797 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9970552389340994 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 221.215576171875 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9927522581688004 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoFiQA2018 |
| type: NanoFiQA2018 |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.28 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.42 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.46 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.5 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.28 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.18 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.136 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.08399999999999999 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.14183333333333334 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.24288888888888888 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.27715873015873016 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.3288730158730159 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.28813286680239514 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.3561904761904763 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.2415362537997973 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 82.86000061035156 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9972852368583202 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 130.93699645996094 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9957100780925245 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoHotpotQA |
| type: NanoHotpotQA |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.78 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.84 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.92 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.98 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.78 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.3733333333333333 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.28400000000000003 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.16 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.39 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.56 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.71 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.8 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.7143331285788386 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.8361904761904762 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.6181181734895289 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 91.9800033569336 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9969864359033833 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 152.01571655273438 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9950194706587794 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoSCIDOCS |
| type: NanoSCIDOCS |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.36 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.58 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.68 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.76 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.36 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.2733333333333333 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.21199999999999997 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.15199999999999997 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.07566666666666666 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.16966666666666666 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.21766666666666665 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.31066666666666665 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.30291194083231554 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.4943888888888889 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.21666464487074008 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 94.30000305175781 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.996910425167035 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 199.64630126953125 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9934589377737524 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoArguAna |
| type: NanoArguAna |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.1 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.34 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.42 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.44 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.1 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.1133333333333333 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.084 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.044000000000000004 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.1 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.34 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.42 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.44 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.2781554838544819 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.22466666666666665 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.2332757160696607 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 189.10000610351562 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9938044687077021 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 164.03329467773438 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9946257357093985 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoSciFact |
| type: NanoSciFact |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.52 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.62 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.64 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.76 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.52 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.21333333333333332 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.14 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.08399999999999999 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.475 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.58 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.615 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.74 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.6020710919940331 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.5799047619047619 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.5551340236204781 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 82.45999908447266 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9972983422094073 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 194.24940490722656 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9936357576532591 |
| name: Corpus Sparsity Ratio |
| - task: |
| type: sparse-information-retrieval |
| name: Sparse Information Retrieval |
| dataset: |
| name: NanoTouche2020 |
| type: NanoTouche2020 |
| metrics: |
| - type: dot_accuracy@1 |
| value: 0.3877551020408163 |
| name: Dot Accuracy@1 |
| - type: dot_accuracy@3 |
| value: 0.7551020408163265 |
| name: Dot Accuracy@3 |
| - type: dot_accuracy@5 |
| value: 0.8367346938775511 |
| name: Dot Accuracy@5 |
| - type: dot_accuracy@10 |
| value: 0.9591836734693877 |
| name: Dot Accuracy@10 |
| - type: dot_precision@1 |
| value: 0.3877551020408163 |
| name: Dot Precision@1 |
| - type: dot_precision@3 |
| value: 0.4693877551020407 |
| name: Dot Precision@3 |
| - type: dot_precision@5 |
| value: 0.4163265306122449 |
| name: Dot Precision@5 |
| - type: dot_precision@10 |
| value: 0.33877551020408164 |
| name: Dot Precision@10 |
| - type: dot_recall@1 |
| value: 0.02376760958202688 |
| name: Dot Recall@1 |
| - type: dot_recall@3 |
| value: 0.08934182714819683 |
| name: Dot Recall@3 |
| - type: dot_recall@5 |
| value: 0.12879429112534482 |
| name: Dot Recall@5 |
| - type: dot_recall@10 |
| value: 0.21659229068805946 |
| name: Dot Recall@10 |
| - type: dot_ndcg@10 |
| value: 0.37622550913382224 |
| name: Dot Ndcg@10 |
| - type: dot_mrr@10 |
| value: 0.5806689342403627 |
| name: Dot Mrr@10 |
| - type: dot_map@100 |
| value: 0.2560796141253303 |
| name: Dot Map@100 |
| - type: query_active_dims |
| value: 79.12245178222656 |
| name: Query Active Dims |
| - type: query_sparsity_ratio |
| value: 0.9974076911151881 |
| name: Query Sparsity Ratio |
| - type: corpus_active_dims |
| value: 135.00782775878906 |
| name: Corpus Active Dims |
| - type: corpus_sparsity_ratio |
| value: 0.9955767044178366 |
| name: Corpus Sparsity Ratio |
| --- |
| |
| # splade-distilbert-base-uncased trained on Quora Duplicates Questions |
|
|
| This is a [SPLADE Sparse Encoder](https://www.sbert.net/docs/sparse_encoder/usage/usage.html) model finetuned from [distilbert/distilbert-base-uncased](https://huggingface.co/distilbert/distilbert-base-uncased) on the [quora-duplicates](https://huggingface.co/datasets/sentence-transformers/quora-duplicates) dataset using the [sentence-transformers](https://www.SBERT.net) library. It maps sentences & paragraphs to a 30522-dimensional sparse vector space and can be used for semantic search and sparse retrieval. |
| ## Model Details |
|
|
| ### Model Description |
| - **Model Type:** SPLADE Sparse Encoder |
| - **Base model:** [distilbert/distilbert-base-uncased](https://huggingface.co/distilbert/distilbert-base-uncased) <!-- at revision 12040accade4e8a0f71eabdb258fecc2e7e948be --> |
| - **Maximum Sequence Length:** 256 tokens |
| - **Output Dimensionality:** 30522 dimensions |
| - **Similarity Function:** Dot Product |
| - **Training Dataset:** |
| - [quora-duplicates](https://huggingface.co/datasets/sentence-transformers/quora-duplicates) |
| - **Language:** en |
| - **License:** apache-2.0 |
|
|
| ### Model Sources |
|
|
| - **Documentation:** [Sentence Transformers Documentation](https://sbert.net) |
| - **Documentation:** [Sparse Encoder Documentation](https://www.sbert.net/docs/sparse_encoder/usage/usage.html) |
| - **Repository:** [Sentence Transformers on GitHub](https://github.com/UKPLab/sentence-transformers) |
| - **Hugging Face:** [Sparse Encoders on Hugging Face](https://huggingface.co/models?library=sentence-transformers&other=sparse-encoder) |
|
|
| ### Full Model Architecture |
|
|
| ``` |
| SparseEncoder( |
| (0): MLMTransformer({'max_seq_length': 256, 'do_lower_case': False}) with MLMTransformer model: DistilBertForMaskedLM |
| (1): SpladePooling({'pooling_strategy': 'max', 'activation_function': 'relu', 'word_embedding_dimension': 30522}) |
| ) |
| ``` |
|
|
| ## Usage |
|
|
| ### Direct Usage (Sentence Transformers) |
|
|
| First install the Sentence Transformers library: |
|
|
| ```bash |
| pip install -U sentence-transformers |
| ``` |
|
|
| Then you can load this model and run inference. |
| ```python |
| from sentence_transformers import SparseEncoder |
| |
| # Download from the 🤗 Hub |
| model = SparseEncoder("arthurbresnu/splade-distilbert-base-uncased-quora-duplicates") |
| # Run inference |
| sentences = [ |
| 'What accomplishments did Hillary Clinton achieve during her time as Secretary of State?', |
| "What are Hillary Clinton's most recognized accomplishments while Secretary of State?", |
| 'What are Hillary Clinton’s qualifications to be President?', |
| ] |
| embeddings = model.encode(sentences) |
| print(embeddings.shape) |
| # (3, 30522) |
| |
| # Get the similarity scores for the embeddings |
| similarities = model.similarity(embeddings, embeddings) |
| print(similarities.shape) |
| # [3, 3] |
| ``` |
|
|
| <!-- |
| ### Direct Usage (Transformers) |
|
|
| <details><summary>Click to see the direct usage in Transformers</summary> |
|
|
| </details> |
| --> |
|
|
| <!-- |
| ### Downstream Usage (Sentence Transformers) |
|
|
| You can finetune this model on your own dataset. |
|
|
| <details><summary>Click to expand</summary> |
|
|
| </details> |
| --> |
|
|
| <!-- |
| ### Out-of-Scope Use |
|
|
| *List how the model may foreseeably be misused and address what users ought not to do with the model.* |
| --> |
|
|
| ## Evaluation |
|
|
| ### Metrics |
|
|
| #### Sparse Binary Classification |
|
|
| * Dataset: `quora_duplicates_dev` |
| * Evaluated with [<code>SparseBinaryClassificationEvaluator</code>](https://sbert.net/docs/package_reference/sparse_encoder/evaluation.html#sentence_transformers.sparse_encoder.evaluation.SparseBinaryClassificationEvaluator) |
|
|
| | Metric | Value | |
| |:-----------------------------|:-----------| |
| | cosine_accuracy | 0.758 | |
| | cosine_accuracy_threshold | 0.8166 | |
| | cosine_f1 | 0.6793 | |
| | cosine_f1_threshold | 0.5696 | |
| | cosine_precision | 0.5488 | |
| | cosine_recall | 0.8913 | |
| | cosine_ap | 0.6888 | |
| | cosine_mcc | 0.5082 | |
| | dot_accuracy | 0.765 | |
| | dot_accuracy_threshold | 51.6699 | |
| | dot_f1 | 0.6762 | |
| | dot_f1_threshold | 46.5249 | |
| | dot_precision | 0.5817 | |
| | dot_recall | 0.8075 | |
| | dot_ap | 0.6336 | |
| | dot_mcc | 0.4996 | |
| | euclidean_accuracy | 0.677 | |
| | euclidean_accuracy_threshold | -14.2724 | |
| | euclidean_f1 | 0.486 | |
| | euclidean_f1_threshold | -0.6445 | |
| | euclidean_precision | 0.3213 | |
| | euclidean_recall | 0.9969 | |
| | euclidean_ap | 0.2033 | |
| | euclidean_mcc | -0.0459 | |
| | manhattan_accuracy | 0.677 | |
| | manhattan_accuracy_threshold | -161.7768 | |
| | manhattan_f1 | 0.486 | |
| | manhattan_f1_threshold | -3.0495 | |
| | manhattan_precision | 0.3213 | |
| | manhattan_recall | 0.9969 | |
| | manhattan_ap | 0.2044 | |
| | manhattan_mcc | -0.0459 | |
| | max_accuracy | 0.765 | |
| | max_accuracy_threshold | 51.6699 | |
| | max_f1 | 0.6793 | |
| | max_f1_threshold | 46.5249 | |
| | max_precision | 0.5817 | |
| | max_recall | 0.9969 | |
| | **max_ap** | **0.6888** | |
| | max_mcc | 0.5082 | |
| | active_dims | 78.3228 | |
| | sparsity_ratio | 0.9974 | |
| |
| #### Sparse Information Retrieval |
| |
| * Datasets: `NanoMSMARCO`, `NanoNQ`, `NanoNFCorpus`, `NanoQuoraRetrieval`, `NanoClimateFEVER`, `NanoDBPedia`, `NanoFEVER`, `NanoFiQA2018`, `NanoHotpotQA`, `NanoMSMARCO`, `NanoNFCorpus`, `NanoNQ`, `NanoQuoraRetrieval`, `NanoSCIDOCS`, `NanoArguAna`, `NanoSciFact` and `NanoTouche2020` |
| * Evaluated with [<code>SparseInformationRetrievalEvaluator</code>](https://sbert.net/docs/package_reference/sparse_encoder/evaluation.html#sentence_transformers.sparse_encoder.evaluation.SparseInformationRetrievalEvaluator) |
| |
| | Metric | NanoMSMARCO | NanoNQ | NanoNFCorpus | NanoQuoraRetrieval | NanoClimateFEVER | NanoDBPedia | NanoFEVER | NanoFiQA2018 | NanoHotpotQA | NanoSCIDOCS | NanoArguAna | NanoSciFact | NanoTouche2020 | |
| |:----------------------|:------------|:-----------|:-------------|:-------------------|:-----------------|:------------|:----------|:-------------|:-------------|:------------|:------------|:------------|:---------------| |
| | dot_accuracy@1 | 0.22 | 0.38 | 0.34 | 0.94 | 0.18 | 0.6 | 0.58 | 0.28 | 0.78 | 0.36 | 0.1 | 0.52 | 0.3878 | |
| | dot_accuracy@3 | 0.42 | 0.54 | 0.5 | 0.98 | 0.32 | 0.84 | 0.76 | 0.42 | 0.84 | 0.58 | 0.34 | 0.62 | 0.7551 | |
| | dot_accuracy@5 | 0.52 | 0.62 | 0.54 | 0.98 | 0.4 | 0.84 | 0.8 | 0.46 | 0.92 | 0.68 | 0.42 | 0.64 | 0.8367 | |
| | dot_accuracy@10 | 0.76 | 0.62 | 0.58 | 0.98 | 0.48 | 0.92 | 0.86 | 0.5 | 0.98 | 0.76 | 0.44 | 0.76 | 0.9592 | |
| | dot_precision@1 | 0.22 | 0.38 | 0.34 | 0.94 | 0.18 | 0.6 | 0.58 | 0.28 | 0.78 | 0.36 | 0.1 | 0.52 | 0.3878 | |
| | dot_precision@3 | 0.14 | 0.18 | 0.3067 | 0.3933 | 0.1067 | 0.5267 | 0.2667 | 0.18 | 0.3733 | 0.2733 | 0.1133 | 0.2133 | 0.4694 | |
| | dot_precision@5 | 0.104 | 0.124 | 0.26 | 0.248 | 0.084 | 0.456 | 0.168 | 0.136 | 0.284 | 0.212 | 0.084 | 0.14 | 0.4163 | |
| | dot_precision@10 | 0.076 | 0.064 | 0.198 | 0.132 | 0.054 | 0.422 | 0.09 | 0.084 | 0.16 | 0.152 | 0.044 | 0.084 | 0.3388 | |
| | dot_recall@1 | 0.22 | 0.36 | 0.0116 | 0.8173 | 0.085 | 0.0457 | 0.5467 | 0.1418 | 0.39 | 0.0757 | 0.1 | 0.475 | 0.0238 | |
| | dot_recall@3 | 0.42 | 0.52 | 0.0606 | 0.928 | 0.1467 | 0.1537 | 0.7467 | 0.2429 | 0.56 | 0.1697 | 0.34 | 0.58 | 0.0893 | |
| | dot_recall@5 | 0.52 | 0.6 | 0.0826 | 0.946 | 0.1783 | 0.1908 | 0.7867 | 0.2772 | 0.71 | 0.2177 | 0.42 | 0.615 | 0.1288 | |
| | dot_recall@10 | 0.76 | 0.61 | 0.098 | 0.97 | 0.215 | 0.2936 | 0.8467 | 0.3289 | 0.8 | 0.3107 | 0.44 | 0.74 | 0.2166 | |
| | **dot_ndcg@10** | **0.4532** | **0.4828** | **0.2467** | **0.9467** | **0.1845** | **0.5071** | **0.707** | **0.2881** | **0.7143** | **0.3029** | **0.2782** | **0.6021** | **0.3762** | |
| | dot_mrr@10 | 0.3601 | 0.4537 | 0.422 | 0.96 | 0.2674 | 0.7147 | 0.6765 | 0.3562 | 0.8362 | 0.4944 | 0.2247 | 0.5799 | 0.5807 | |
| | dot_map@100 | 0.3733 | 0.4454 | 0.094 | 0.9291 | 0.1476 | 0.3907 | 0.6651 | 0.2415 | 0.6181 | 0.2167 | 0.2333 | 0.5551 | 0.2561 | |
| | query_active_dims | 74.76 | 74.74 | 79.7 | 76.58 | 89.86 | 69.52 | 89.88 | 82.86 | 91.98 | 94.3 | 189.1 | 82.46 | 79.1225 | |
| | query_sparsity_ratio | 0.9976 | 0.9976 | 0.9974 | 0.9975 | 0.9971 | 0.9977 | 0.9971 | 0.9973 | 0.997 | 0.9969 | 0.9938 | 0.9973 | 0.9974 | |
| | corpus_active_dims | 103.0652 | 141.3148 | 202.1727 | 77.5906 | 221.7553 | 135.9335 | 221.2156 | 130.937 | 152.0157 | 199.6463 | 164.0333 | 194.2494 | 135.0078 | |
| | corpus_sparsity_ratio | 0.9966 | 0.9954 | 0.9934 | 0.9975 | 0.9927 | 0.9955 | 0.9928 | 0.9957 | 0.995 | 0.9935 | 0.9946 | 0.9936 | 0.9956 | |
| |
| #### Sparse Nano BEIR |
| |
| * Dataset: `NanoBEIR_mean` |
| * Evaluated with [<code>SparseNanoBEIREvaluator</code>](https://sbert.net/docs/package_reference/sparse_encoder/evaluation.html#sentence_transformers.sparse_encoder.evaluation.SparseNanoBEIREvaluator) with these parameters: |
| ```json |
| { |
| "dataset_names": [ |
| "msmarco", |
| "nq", |
| "nfcorpus", |
| "quoraretrieval" |
| ] |
| } |
| ``` |
|
|
| | Metric | Value | |
| |:----------------------|:-----------| |
| | dot_accuracy@1 | 0.47 | |
| | dot_accuracy@3 | 0.61 | |
| | dot_accuracy@5 | 0.665 | |
| | dot_accuracy@10 | 0.735 | |
| | dot_precision@1 | 0.47 | |
| | dot_precision@3 | 0.255 | |
| | dot_precision@5 | 0.184 | |
| | dot_precision@10 | 0.1175 | |
| | dot_recall@1 | 0.3522 | |
| | dot_recall@3 | 0.4821 | |
| | dot_recall@5 | 0.5372 | |
| | dot_recall@10 | 0.6095 | |
| | **dot_ndcg@10** | **0.5324** | |
| | dot_mrr@10 | 0.5489 | |
| | dot_map@100 | 0.4605 | |
| | query_active_dims | 76.445 | |
| | query_sparsity_ratio | 0.9975 | |
| | corpus_active_dims | 122.7978 | |
| | corpus_sparsity_ratio | 0.996 | |
| |
| #### Sparse Nano BEIR |
| |
| * Dataset: `NanoBEIR_mean` |
| * Evaluated with [<code>SparseNanoBEIREvaluator</code>](https://sbert.net/docs/package_reference/sparse_encoder/evaluation.html#sentence_transformers.sparse_encoder.evaluation.SparseNanoBEIREvaluator) with these parameters: |
| ```json |
| { |
| "dataset_names": [ |
| "climatefever", |
| "dbpedia", |
| "fever", |
| "fiqa2018", |
| "hotpotqa", |
| "msmarco", |
| "nfcorpus", |
| "nq", |
| "quoraretrieval", |
| "scidocs", |
| "arguana", |
| "scifact", |
| "touche2020" |
| ] |
| } |
| ``` |
| |
| | Metric | Value | |
| |:----------------------|:-----------| |
| | dot_accuracy@1 | 0.436 | |
| | dot_accuracy@3 | 0.6089 | |
| | dot_accuracy@5 | 0.6659 | |
| | dot_accuracy@10 | 0.7384 | |
| | dot_precision@1 | 0.436 | |
| | dot_precision@3 | 0.2725 | |
| | dot_precision@5 | 0.2089 | |
| | dot_precision@10 | 0.1461 | |
| | dot_recall@1 | 0.2533 | |
| | dot_recall@3 | 0.3813 | |
| | dot_recall@5 | 0.4364 | |
| | dot_recall@10 | 0.51 | |
| | **dot_ndcg@10** | **0.4685** | |
| | dot_mrr@10 | 0.5328 | |
| | dot_map@100 | 0.3974 | |
| | query_active_dims | 90.3914 | |
| | query_sparsity_ratio | 0.997 | |
| | corpus_active_dims | 152.3669 | |
| | corpus_sparsity_ratio | 0.995 | |
|
|
| <!-- |
| ## Bias, Risks and Limitations |
|
|
| *What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.* |
| --> |
|
|
| <!-- |
| ### Recommendations |
|
|
| *What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.* |
| --> |
|
|
| ## Training Details |
|
|
| ### Training Dataset |
|
|
| #### quora-duplicates |
|
|
| * Dataset: [quora-duplicates](https://huggingface.co/datasets/sentence-transformers/quora-duplicates) at [451a485](https://huggingface.co/datasets/sentence-transformers/quora-duplicates/tree/451a4850bd141edb44ade1b5828c259abd762cdb) |
| * Size: 99,000 training samples |
| * Columns: <code>anchor</code>, <code>positive</code>, and <code>negative</code> |
| * Approximate statistics based on the first 1000 samples: |
| | | anchor | positive | negative | |
| |:--------|:---------------------------------------------------------------------------------|:----------------------------------------------------------------------------------|:----------------------------------------------------------------------------------| |
| | type | string | string | string | |
| | details | <ul><li>min: 6 tokens</li><li>mean: 14.1 tokens</li><li>max: 39 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 13.83 tokens</li><li>max: 41 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 15.21 tokens</li><li>max: 75 tokens</li></ul> | |
| * Samples: |
| | anchor | positive | negative | |
| |:----------------------------------------------------------------------|:---------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| |
| | <code>What are the best GMAT coaching institutes in Delhi NCR?</code> | <code>Which are the best GMAT coaching institutes in Delhi/NCR?</code> | <code>What are the best GMAT coaching institutes in Delhi-Noida Area?</code> | |
| | <code>Is a third world war coming?</code> | <code>Is World War 3 more imminent than expected?</code> | <code>Since the UN is unable to control terrorism and groups like ISIS, al-Qaeda and countries that promote terrorism (even though it consumed those countries), can we assume that the world is heading towards World War III?</code> | |
| | <code>Should I build iOS or Android apps first?</code> | <code>Should people choose Android or iOS first to build their App?</code> | <code>How much more effort is it to build your app on both iOS and Android?</code> | |
| * Loss: [<code>SpladeLoss</code>](https://sbert.net/docs/package_reference/sparse_encoder/losses.html#spladeloss) with these parameters: |
| ```json |
| { |
| "loss": "SparseMultipleNegativesRankingLoss(scale=1.0, similarity_fct='dot_score')", |
| "lambda_corpus": 3e-05, |
| "lambda_query": 5e-05 |
| } |
| ``` |
|
|
| ### Evaluation Dataset |
|
|
| #### quora-duplicates |
|
|
| * Dataset: [quora-duplicates](https://huggingface.co/datasets/sentence-transformers/quora-duplicates) at [451a485](https://huggingface.co/datasets/sentence-transformers/quora-duplicates/tree/451a4850bd141edb44ade1b5828c259abd762cdb) |
| * Size: 1,000 evaluation samples |
| * Columns: <code>anchor</code>, <code>positive</code>, and <code>negative</code> |
| * Approximate statistics based on the first 1000 samples: |
| | | anchor | positive | negative | |
| |:--------|:----------------------------------------------------------------------------------|:----------------------------------------------------------------------------------|:----------------------------------------------------------------------------------| |
| | type | string | string | string | |
| | details | <ul><li>min: 6 tokens</li><li>mean: 14.05 tokens</li><li>max: 40 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 14.14 tokens</li><li>max: 44 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 14.56 tokens</li><li>max: 60 tokens</li></ul> | |
| * Samples: |
| | anchor | positive | negative | |
| |:-------------------------------------------------------------------|:------------------------------------------------------------|:-----------------------------------------------------------------| |
| | <code>What happens if we use petrol in diesel vehicles?</code> | <code>Why can't we use petrol in diesel?</code> | <code>Why are diesel engines noisier than petrol engines?</code> | |
| | <code>Why is Saltwater taffy candy imported in Switzerland?</code> | <code>Why is Saltwater taffy candy imported in Laos?</code> | <code>Is salt a consumer product?</code> | |
| | <code>Which is your favourite film in 2016?</code> | <code>What movie is the best movie of 2016?</code> | <code>What will the best movie of 2017 be?</code> | |
| * Loss: [<code>SpladeLoss</code>](https://sbert.net/docs/package_reference/sparse_encoder/losses.html#spladeloss) with these parameters: |
| ```json |
| { |
| "loss": "SparseMultipleNegativesRankingLoss(scale=1.0, similarity_fct='dot_score')", |
| "lambda_corpus": 3e-05, |
| "lambda_query": 5e-05 |
| } |
| ``` |
|
|
| ### Training Hyperparameters |
| #### Non-Default Hyperparameters |
|
|
| - `eval_strategy`: steps |
| - `per_device_train_batch_size`: 12 |
| - `per_device_eval_batch_size`: 12 |
| - `learning_rate`: 2e-05 |
| - `num_train_epochs`: 1 |
| - `bf16`: True |
| - `load_best_model_at_end`: True |
| - `batch_sampler`: no_duplicates |
| |
| #### All Hyperparameters |
| <details><summary>Click to expand</summary> |
| |
| - `overwrite_output_dir`: False |
| - `do_predict`: False |
| - `eval_strategy`: steps |
| - `prediction_loss_only`: True |
| - `per_device_train_batch_size`: 12 |
| - `per_device_eval_batch_size`: 12 |
| - `per_gpu_train_batch_size`: None |
| - `per_gpu_eval_batch_size`: None |
| - `gradient_accumulation_steps`: 1 |
| - `eval_accumulation_steps`: None |
| - `torch_empty_cache_steps`: None |
| - `learning_rate`: 2e-05 |
| - `weight_decay`: 0.0 |
| - `adam_beta1`: 0.9 |
| - `adam_beta2`: 0.999 |
| - `adam_epsilon`: 1e-08 |
| - `max_grad_norm`: 1.0 |
| - `num_train_epochs`: 1 |
| - `max_steps`: -1 |
| - `lr_scheduler_type`: linear |
| - `lr_scheduler_kwargs`: {} |
| - `warmup_ratio`: 0.0 |
| - `warmup_steps`: 0 |
| - `log_level`: passive |
| - `log_level_replica`: warning |
| - `log_on_each_node`: True |
| - `logging_nan_inf_filter`: True |
| - `save_safetensors`: True |
| - `save_on_each_node`: False |
| - `save_only_model`: False |
| - `restore_callback_states_from_checkpoint`: False |
| - `no_cuda`: False |
| - `use_cpu`: False |
| - `use_mps_device`: False |
| - `seed`: 42 |
| - `data_seed`: None |
| - `jit_mode_eval`: False |
| - `use_ipex`: False |
| - `bf16`: True |
| - `fp16`: False |
| - `fp16_opt_level`: O1 |
| - `half_precision_backend`: auto |
| - `bf16_full_eval`: False |
| - `fp16_full_eval`: False |
| - `tf32`: None |
| - `local_rank`: 0 |
| - `ddp_backend`: None |
| - `tpu_num_cores`: None |
| - `tpu_metrics_debug`: False |
| - `debug`: [] |
| - `dataloader_drop_last`: False |
| - `dataloader_num_workers`: 0 |
| - `dataloader_prefetch_factor`: None |
| - `past_index`: -1 |
| - `disable_tqdm`: False |
| - `remove_unused_columns`: True |
| - `label_names`: None |
| - `load_best_model_at_end`: True |
| - `ignore_data_skip`: False |
| - `fsdp`: [] |
| - `fsdp_min_num_params`: 0 |
| - `fsdp_config`: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False} |
| - `tp_size`: 0 |
| - `fsdp_transformer_layer_cls_to_wrap`: None |
| - `accelerator_config`: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None} |
| - `deepspeed`: None |
| - `label_smoothing_factor`: 0.0 |
| - `optim`: adamw_torch |
| - `optim_args`: None |
| - `adafactor`: False |
| - `group_by_length`: False |
| - `length_column_name`: length |
| - `ddp_find_unused_parameters`: None |
| - `ddp_bucket_cap_mb`: None |
| - `ddp_broadcast_buffers`: False |
| - `dataloader_pin_memory`: True |
| - `dataloader_persistent_workers`: False |
| - `skip_memory_metrics`: True |
| - `use_legacy_prediction_loop`: False |
| - `push_to_hub`: False |
| - `resume_from_checkpoint`: None |
| - `hub_model_id`: None |
| - `hub_strategy`: every_save |
| - `hub_private_repo`: None |
| - `hub_always_push`: False |
| - `gradient_checkpointing`: False |
| - `gradient_checkpointing_kwargs`: None |
| - `include_inputs_for_metrics`: False |
| - `include_for_metrics`: [] |
| - `eval_do_concat_batches`: True |
| - `fp16_backend`: auto |
| - `push_to_hub_model_id`: None |
| - `push_to_hub_organization`: None |
| - `mp_parameters`: |
| - `auto_find_batch_size`: False |
| - `full_determinism`: False |
| - `torchdynamo`: None |
| - `ray_scope`: last |
| - `ddp_timeout`: 1800 |
| - `torch_compile`: False |
| - `torch_compile_backend`: None |
| - `torch_compile_mode`: None |
| - `dispatch_batches`: None |
| - `split_batches`: None |
| - `include_tokens_per_second`: False |
| - `include_num_input_tokens_seen`: False |
| - `neftune_noise_alpha`: None |
| - `optim_target_modules`: None |
| - `batch_eval_metrics`: False |
| - `eval_on_start`: False |
| - `use_liger_kernel`: False |
| - `eval_use_gather_object`: False |
| - `average_tokens_across_devices`: False |
| - `prompts`: None |
| - `batch_sampler`: no_duplicates |
| - `multi_dataset_batch_sampler`: proportional |
|
|
| </details> |
|
|
| ### Training Logs |
| | Epoch | Step | Training Loss | Validation Loss | quora_duplicates_dev_max_ap | NanoMSMARCO_dot_ndcg@10 | NanoNQ_dot_ndcg@10 | NanoNFCorpus_dot_ndcg@10 | NanoQuoraRetrieval_dot_ndcg@10 | NanoBEIR_mean_dot_ndcg@10 | NanoClimateFEVER_dot_ndcg@10 | NanoDBPedia_dot_ndcg@10 | NanoFEVER_dot_ndcg@10 | NanoFiQA2018_dot_ndcg@10 | NanoHotpotQA_dot_ndcg@10 | NanoSCIDOCS_dot_ndcg@10 | NanoArguAna_dot_ndcg@10 | NanoSciFact_dot_ndcg@10 | NanoTouche2020_dot_ndcg@10 | |
| |:-------:|:--------:|:-------------:|:---------------:|:---------------------------:|:-----------------------:|:------------------:|:------------------------:|:------------------------------:|:-------------------------:|:----------------------------:|:-----------------------:|:---------------------:|:------------------------:|:------------------------:|:-----------------------:|:-----------------------:|:-----------------------:|:--------------------------:| |
| | 0.0242 | 200 | 8.3389 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.0485 | 400 | 0.4397 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.0727 | 600 | 0.3737 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.0970 | 800 | 0.2666 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.1212 | 1000 | 0.288 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.1455 | 1200 | 0.1977 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.1697 | 1400 | 0.2707 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.1939 | 1600 | 0.1951 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.2 | 1650 | - | 0.1669 | 0.6472 | 0.3052 | 0.2793 | 0.1711 | 0.9281 | 0.4209 | - | - | - | - | - | - | - | - | - | |
| | 0.2182 | 1800 | 0.2178 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.2424 | 2000 | 0.2174 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.2667 | 2200 | 0.1832 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.2909 | 2400 | 0.1879 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.3152 | 2600 | 0.1723 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.3394 | 2800 | 0.1543 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.3636 | 3000 | 0.1559 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.3879 | 3200 | 0.1575 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.4 | 3300 | - | 0.1149 | 0.6749 | 0.3894 | 0.4467 | 0.2360 | 0.9292 | 0.5003 | - | - | - | - | - | - | - | - | - | |
| | 0.4121 | 3400 | 0.1395 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.4364 | 3600 | 0.1596 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.4606 | 3800 | 0.1595 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.4848 | 4000 | 0.1211 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.5091 | 4200 | 0.1163 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.5333 | 4400 | 0.1182 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.5576 | 4600 | 0.1337 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.5818 | 4800 | 0.1362 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.6 | 4950 | - | 0.1001 | 0.6802 | 0.4093 | 0.4269 | 0.2341 | 0.9365 | 0.5017 | - | - | - | - | - | - | - | - | - | |
| | 0.6061 | 5000 | 0.1112 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.6303 | 5200 | 0.1064 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.6545 | 5400 | 0.119 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.6788 | 5600 | 0.1077 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.7030 | 5800 | 0.1398 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.7273 | 6000 | 0.09 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.7515 | 6200 | 0.0903 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.7758 | 6400 | 0.1082 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.8 | 6600 | 0.1122 | 0.0901 | 0.6941 | 0.4451 | 0.4757 | 0.2542 | 0.9411 | 0.5290 | - | - | - | - | - | - | - | - | - | |
| | 0.8242 | 6800 | 0.0708 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.8485 | 7000 | 0.1291 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.8727 | 7200 | 0.1165 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.8970 | 7400 | 0.0735 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.9212 | 7600 | 0.0775 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.9455 | 7800 | 0.0945 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.9697 | 8000 | 0.0912 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | 0.9939 | 8200 | 0.104 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
| | **1.0** | **8250** | **-** | **0.0686** | **0.6888** | **0.4532** | **0.4828** | **0.2467** | **0.9467** | **0.5324** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | |
| | -1 | -1 | - | - | - | 0.4532 | 0.4828 | 0.2467 | 0.9467 | 0.4685 | 0.1845 | 0.5071 | 0.7070 | 0.2881 | 0.7143 | 0.3029 | 0.2782 | 0.6021 | 0.3762 | |
| |
| * The bold row denotes the saved checkpoint. |
| |
| ### Environmental Impact |
| Carbon emissions were measured using [CodeCarbon](https://github.com/mlco2/codecarbon). |
| - **Energy Consumed**: 0.025 kWh |
| - **Carbon Emitted**: 0.001 kg of CO2 |
| - **Hours Used**: 0.222 hours |
| |
| ### Training Hardware |
| - **On Cloud**: No |
| - **GPU Model**: 1 x NVIDIA GeForce RTX 3070 Ti Laptop GPU |
| - **CPU Model**: AMD Ryzen 9 6900HX with Radeon Graphics |
| - **RAM Size**: 30.61 GB |
| |
| ### Framework Versions |
| - Python: 3.12.9 |
| - Sentence Transformers: 4.2.0.dev0 |
| - Transformers: 4.50.3 |
| - PyTorch: 2.6.0+cu124 |
| - Accelerate: 1.6.0 |
| - Datasets: 3.5.0 |
| - Tokenizers: 0.21.1 |
| |
| ## Citation |
| |
| ### BibTeX |
| |
| #### Sentence Transformers |
| ```bibtex |
| @inproceedings{reimers-2019-sentence-bert, |
| title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks", |
| author = "Reimers, Nils and Gurevych, Iryna", |
| booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing", |
| month = "11", |
| year = "2019", |
| publisher = "Association for Computational Linguistics", |
| url = "https://arxiv.org/abs/1908.10084", |
| } |
| ``` |
| |
| #### SpladeLoss |
| ```bibtex |
| @misc{formal2022distillationhardnegativesampling, |
| title={From Distillation to Hard Negative Sampling: Making Sparse Neural IR Models More Effective}, |
| author={Thibault Formal and Carlos Lassance and Benjamin Piwowarski and Stéphane Clinchant}, |
| year={2022}, |
| eprint={2205.04733}, |
| archivePrefix={arXiv}, |
| primaryClass={cs.IR}, |
| url={https://arxiv.org/abs/2205.04733}, |
| } |
| ``` |
| |
| #### SparseMultipleNegativesRankingLoss |
| ```bibtex |
| @misc{henderson2017efficient, |
| title={Efficient Natural Language Response Suggestion for Smart Reply}, |
| author={Matthew Henderson and Rami Al-Rfou and Brian Strope and Yun-hsuan Sung and Laszlo Lukacs and Ruiqi Guo and Sanjiv Kumar and Balint Miklos and Ray Kurzweil}, |
| year={2017}, |
| eprint={1705.00652}, |
| archivePrefix={arXiv}, |
| primaryClass={cs.CL} |
| } |
| ``` |
| |
| #### FlopsLoss |
| ```bibtex |
| @article{paria2020minimizing, |
| title={Minimizing flops to learn efficient sparse representations}, |
| author={Paria, Biswajit and Yeh, Chih-Kuan and Yen, Ian EH and Xu, Ning and Ravikumar, Pradeep and P{'o}czos, Barnab{'a}s}, |
| journal={arXiv preprint arXiv:2004.05665}, |
| year={2020} |
| } |
| ``` |
| |
| <!-- |
| ## Glossary |
| |
| *Clearly define terms in order to be accessible across audiences.* |
| --> |
| |
| <!-- |
| ## Model Card Authors |
| |
| *Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.* |
| --> |
| |
| <!-- |
| ## Model Card Contact |
| |
| *Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.* |
| --> |
