Dutch NLG 🇳🇱

gpt-4-0613 (few-shot, val)

unknown

100

8316

True

597 ± 197 / 93 ± 33

1.21

73.35 ± 2.61 / 56.00 ± 2.82

18.92 ± 2.78 / 40.80 ± 2.43

76.70 ± 2.39 / 88.16 ± 1.21

55.03 ± 1.96 / 76.47 ± 1.22

69.97 ± 0.41 / 22.66 ± 0.68

70.71 ± 2.96 / 78.12 ± 2.24

90.07 ± 1.29 / 92.54 ± 0.98

12.10.0

meta-llama/Meta-Llama-3-70B (few-shot, val)

70554

128

8317

True

312 ± 55 / 177 ± 51

1.46

72.91 ± 3.24 / 68.06 ± 4.62

19.08 ± 3.37 / 42.04 ± 2.31

54.33 ± 3.49 / 75.54 ± 2.31

63.99 ± 2.07 / 77.63 ± 1.16

70.41 ± 0.97 / 25.41 ± 1.66

65.33 ± 1.92 / 73.98 ± 1.56

71.06 ± 2.82 / 78.16 ± 2.22

12.7.0

gpt-4-1106-preview (few-shot, val)

unknown

100

128000

True

576 ± 221 / 81 ± 28

1.50

66.44 ± 2.18 / 56.97 ± 2.87

14.22 ± 3.26 / 33.41 ± 3.24

72.30 ± 2.26 / 85.96 ± 1.13

57.81 ± 1.23 / 74.51 ± 0.62

67.13 ± 0.50 / 17.61 ± 0.76

70.04 ± 1.91 / 77.58 ± 1.42

88.29 ± 2.26 / 91.17 ± 1.70

12.3.2

12.10.2

12.3.2

12.10.2

meta-llama/Meta-Llama-3-70B-Instruct (few-shot, val)

70554

128

8317

True

1,673 ± 583 / 275 ± 85

1.62

74.64 ± 3.67 / 71.84 ± 4.01

18.90 ± 2.04 / 41.93 ± 1.60

49.54 ± 4.22 / 74.03 ± 2.52

44.77 ± 1.67 / 71.44 ± 1.30

70.28 ± 0.65 / 22.72 ± 0.95

65.80 ± 3.16 / 74.38 ± 2.37

76.21 ± 1.36 / 82.15 ± 1.01

12.7.0

gpt-4o-2024-05-13 (few-shot, val)

unknown

200

128000

True

916 ± 329 / 114 ± 38

1.66

76.75 ± 3.44 / 61.13 ± 4.40

10.80 ± 2.24 / 32.52 ± 2.18

56.26 ± 4.51 / 73.83 ± 3.26

55.55 ± 2.54 / 76.28 ± 1.13

66.86 ± 0.46 / 16.93 ± 0.91

73.11 ± 1.87 / 79.69 ± 1.46

92.69 ± 1.46 / 94.53 ± 1.09

12.10.0

12.10.2

12.10.0

12.10.2

152334H/miqu-1-70b-sf (few-shot, val)

68977

32

32889

True

2,126 ± 676 / 319 ± 104

1.75

67.00 ± 3.69 / 56.41 ± 4.29

15.33 ± 4.14 / 36.14 ± 2.91

55.48 ± 4.37 / 77.55 ± 2.24

61.02 ± 1.67 / 76.87 ± 1.15

68.95 ± 0.75 / 21.31 ± 1.20

54.36 ± 1.61 / 65.70 ± 1.26

70.26 ± 3.64 / 77.62 ± 2.82

12.7.0

google/gemma-2-27b-it (few-shot)

27227

256

8320

True

1,516 ± 257 / 480 ± 148

1.84

65.20 ± 1.76 / 53.16 ± 2.84

14.80 ± 0.93 / 36.86 ± 1.08

59.02 ± 1.53 / 79.44 ± 0.78

59.60 ± 1.51 / 74.99 ± 0.59

64.03 ± 1.25 / 20.76 ± 1.31

59.71 ± 0.83 / 69.70 ± 0.64

70.70 ± 1.31 / 77.81 ± 0.99

13.0.0

google/gemma-2-9b-it (few-shot)

9242

256

8320

True

2,062 ± 397 / 589 ± 178

2.07

52.62 ± 2.15 / 39.41 ± 1.72

11.78 ± 1.31 / 32.80 ± 0.87

59.23 ± 1.58 / 79.42 ± 0.88

55.78 ± 0.87 / 72.71 ± 0.70

67.39 ± 0.71 / 20.04 ± 1.05

53.99 ± 0.77 / 65.31 ± 0.62

65.58 ± 1.18 / 73.75 ± 0.94

13.0.0

upstage/SOLAR-10.7B-v1.0 (few-shot)

10732

32

4096

True

3,780 ± 906 / 799 ± 261

2.09

65.37 ± 1.61 / 46.10 ± 1.53

11.93 ± 1.80 / 34.67 ± 2.84

41.67 ± 1.53 / 69.81 ± 1.38

67.75 ± 0.62 / 78.01 ± 0.45

68.83 ± 0.35 / 26.12 ± 0.52

40.18 ± 0.57 / 55.06 ± 0.44

65.34 ± 1.05 / 73.48 ± 0.83

12.5.3

google/gemma-2-9b (few-shot)

9242

256

8320

True

2,038 ± 406 / 566 ± 172

2.11

57.13 ± 2.73 / 36.21 ± 1.71

17.43 ± 2.17 / 40.83 ± 1.50

31.39 ± 5.53 / 56.70 ± 5.97

59.33 ± 1.35 / 73.56 ± 0.52

66.59 ± 1.21 / 21.08 ± 1.63

53.71 ± 0.86 / 65.15 ± 0.63

53.26 ± 3.38 / 63.65 ± 2.84

13.0.0

mistralai/Ministral-8B-Instruct-2410 (few-shot)

8020

131

32768

True

5,821 ± 1,090 / 1,561 ± 506

2.14

66.51 ± 1.38 / 52.40 ± 2.62

13.55 ± 2.14 / 37.36 ± 1.32

34.46 ± 2.79 / 65.61 ± 2.58

59.23 ± 1.16 / 72.56 ± 0.80

68.87 ± 0.72 / 21.06 ± 1.13

40.81 ± 1.05 / 55.44 ± 0.80

60.68 ± 1.04 / 70.24 ± 0.85

13.0.0

Nexusflow/Starling-LM-7B-beta (few-shot)

7242

32

8192

False

5,876 ± 1,021 / 1,677 ± 546

2.15

64.47 ± 2.31 / 40.89 ± 2.81

13.83 ± 1.91 / 41.53 ± 1.23

45.69 ± 1.76 / 72.13 ± 1.39

58.03 ± 1.37 / 73.17 ± 0.58

69.34 ± 0.77 / 22.22 ± 0.98

37.60 ± 0.83 / 53.13 ± 0.62

54.33 ± 0.84 / 65.41 ± 0.59

12.5.2

gpt-3.5-turbo-0613 (few-shot, val)

unknown

100

4095

True

921 ± 293 / 113 ± 37

2.15

68.96 ± 3.80 / 58.45 ± 3.71

8.81 ± 3.30 / 30.88 ± 2.25

58.95 ± 4.48 / 78.64 ± 2.32

55.57 ± 2.33 / 68.26 ± 1.85

69.13 ± 0.41 / 21.32 ± 0.75

42.28 ± 2.88 / 56.45 ± 2.26

61.52 ± 2.69 / 70.62 ± 2.20

0.0.0

meta-llama/Llama-2-70b-hf (few-shot, val)

68977

32

4221

True

1,892 ± 650 / 318 ± 105

2.21

66.50 ± 3.72 / 57.66 ± 3.78

7.82 ± 4.30 / 34.91 ± 2.53

49.55 ± 4.95 / 73.43 ± 3.38

65.26 ± 1.55 / 77.36 ± 1.41

67.28 ± 1.07 / 21.33 ± 1.39

49.41 ± 2.21 / 61.91 ± 1.74

54.50 ± 4.84 / 65.70 ± 3.64

12.7.0

CohereForAI/aya-expanse-8b (few-shot)

8028

256

8192

False

5,581 ± 1,066 / 1,471 ± 483

2.25

62.07 ± 1.67 / 37.68 ± 1.28

11.09 ± 1.67 / 31.37 ± 1.35

35.14 ± 2.33 / 66.66 ± 1.50

49.15 ± 1.48 / 68.82 ± 0.68

74.40 ± 0.20 / 31.66 ± 0.46

46.76 ± 0.71 / 60.01 ± 0.54

60.36 ± 0.88 / 70.23 ± 0.64

13.0.0

ibm-granite/granite-3.0-8b-base (few-shot)

8171

49

4224

True

5,116 ± 943 / 1,436 ± 472

2.28

53.21 ± 1.37 / 42.13 ± 2.58

13.72 ± 2.18 / 39.38 ± 2.08

40.85 ± 4.16 / 67.34 ± 3.66

62.41 ± 0.99 / 74.19 ± 0.56

66.70 ± 0.37 / 19.34 ± 0.85

39.74 ± 0.95 / 54.61 ± 0.75

54.37 ± 0.74 / 65.48 ± 0.65

13.0.0

mistralai/Mistral-Nemo-Instruct-2407 (few-shot)

12248

131

1024128

True

7,095 ± 2,193 / 1,063 ± 344

2.31

66.57 ± 1.86 / 48.40 ± 2.67

10.10 ± 1.55 / 33.62 ± 2.04

40.31 ± 2.25 / 69.53 ± 1.51

59.99 ± 0.95 / 74.24 ± 0.63

68.66 ± 0.68 / 20.75 ± 1.15

42.50 ± 0.84 / 56.53 ± 0.62

48.51 ± 1.44 / 60.59 ± 1.34

13.0.0

meta-llama/Llama-2-70b-chat-hf (few-shot, val)

68977

32

4221

True

1,979 ± 621 / 320 ± 105

2.36

64.00 ± 3.52 / 48.94 ± 3.83

13.30 ± 3.75 / 30.50 ± 2.48

30.88 ± 4.62 / 59.62 ± 4.50

54.14 ± 1.55 / 70.96 ± 1.01

68.71 ± 0.70 / 19.82 ± 0.90

45.47 ± 2.07 / 59.14 ± 1.69

42.91 ± 3.26 / 57.30 ± 2.39

12.7.0

meta-llama/Llama-3.1-8B-Instruct (few-shot)

8030

128

131200

True

2,411 ± 465 / 686 ± 215

2.39

69.76 ± 1.36 / 57.66 ± 1.36

15.51 ± 1.59 / 39.71 ± 1.21

37.58 ± 3.42 / 66.98 ± 2.22

41.26 ± 2.09 / 65.63 ± 0.90

68.84 ± 0.62 / 21.41 ± 1.20

40.98 ± 0.71 / 55.51 ± 0.59

39.98 ± 1.59 / 54.32 ± 1.27

13.0.0

gpt-4o-mini-2024-07-18 (few-shot)

unknown

200

128126

True

1,171 ± 378 / 120 ± 39

2.42

68.20 ± 1.93 / 49.92 ± 2.53

5.65 ± 1.80 / 14.84 ± 2.18

31.15 ± 7.70 / 52.16 ± 7.03

53.17 ± 1.82 / 72.08 ± 0.90

67.62 ± 0.45 / 18.21 ± 0.81

54.56 ± 1.84 / 64.78 ± 1.89

65.27 ± 1.51 / 72.80 ± 1.37

12.11.0

senseable/WestLake-7B-v2 (few-shot)

7242

32

32768

False

5,993 ± 1,028 / 1,742 ± 561

2.42

64.25 ± 2.23 / 46.52 ± 1.72

13.66 ± 1.99 / 39.45 ± 1.52

28.59 ± 1.48 / 61.24 ± 1.46

49.64 ± 0.86 / 68.04 ± 0.55

68.66 ± 0.57 / 19.51 ± 0.67

35.37 ± 1.30 / 51.43 ± 0.96

47.50 ± 1.75 / 60.41 ± 1.32

12.6.1

CohereForAI/aya-23-8B (few-shot)

8028

256

8192

False

5,608 ± 1,062 / 1,472 ± 479

2.45

60.81 ± 1.94 / 46.59 ± 3.32

7.90 ± 1.63 / 24.82 ± 0.95

31.12 ± 2.35 / 64.29 ± 1.88

63.00 ± 1.23 / 74.60 ± 0.67

72.90 ± 0.24 / 30.63 ± 0.42

32.37 ± 1.09 / 48.63 ± 0.82

53.32 ± 1.89 / 64.30 ± 1.72

13.0.0

meta-llama/Llama-3.1-8B (few-shot)

8030

128

131200

True

2,439 ± 459 / 703 ± 219

2.45

64.79 ± 1.96 / 45.48 ± 2.24

11.95 ± 2.83 / 37.12 ± 2.19

32.97 ± 2.68 / 58.52 ± 2.92

63.89 ± 1.06 / 74.73 ± 1.02

66.29 ± 1.29 / 20.14 ± 1.64

38.44 ± 1.33 / 53.68 ± 1.01

30.88 ± 2.27 / 47.18 ± 1.81

12.11.0

13.0.0

utter-project/EuroLLM-9B-Instruct (few-shot)

9152

128

4096

False

5,763 ± 1,178 / 1,390 ± 446

2.47

43.06 ± 1.89 / 30.50 ± 1.45

11.95 ± 2.25 / 33.40 ± 1.41

40.85 ± 3.31 / 68.94 ± 2.23

63.42 ± 0.63 / 74.62 ± 0.52

69.33 ± 0.60 / 23.01 ± 1.18

41.90 ± 1.02 / 56.09 ± 0.76

36.69 ± 1.64 / 50.85 ± 1.65

13.1.0

meta-llama/Meta-Llama-3-8B-Instruct (few-shot)

8030

128

8192

True

4,909 ± 1,215 / 978 ± 319

2.52

68.72 ± 1.81 / 54.89 ± 2.10

14.67 ± 2.51 / 41.36 ± 2.04

32.91 ± 2.56 / 64.93 ± 1.97

45.36 ± 1.31 / 67.50 ± 0.69

67.62 ± 0.82 / 18.19 ± 1.14

36.18 ± 1.31 / 51.68 ± 1.05

33.91 ± 1.71 / 48.01 ± 1.47

12.6.1

skole-gpt-mixtral (few-shot)

unknown

32

32768

False

3,583 ± 977 / 686 ± 231

2.55

62.16 ± 1.09 / 45.76 ± 2.07

8.92 ± 1.08 / 24.28 ± 0.76

32.76 ± 2.94 / 65.17 ± 2.79

56.87 ± 0.92 / 72.57 ± 0.85

68.39 ± 0.73 / 20.83 ± 1.21

46.17 ± 0.64 / 59.57 ± 0.48

33.31 ± 1.83 / 49.12 ± 1.64

13.0.0

yhavinga/Boreas-7B-chat (few-shot)

7242

32

32768

False

2,913 ± 459 / 1,129 ± 342

2.61

60.22 ± 1.55 / 38.72 ± 1.45

11.97 ± 1.80 / 35.17 ± 3.03

30.94 ± 4.81 / 62.66 ± 3.36

52.19 ± 1.96 / 67.52 ± 1.56

66.60 ± 1.93 / 20.00 ± 1.35

35.34 ± 1.32 / 51.45 ± 0.99

31.47 ± 2.90 / 47.64 ± 2.52

12.6.1

meta-llama/Meta-Llama-3-8B (few-shot)

8030

128

8192

True

4,687 ± 1,121 / 967 ± 313

2.63

62.26 ± 2.20 / 42.41 ± 2.02

10.45 ± 2.69 / 33.45 ± 1.99

30.30 ± 3.94 / 62.28 ± 2.89

62.99 ± 1.00 / 73.73 ± 0.98

65.17 ± 1.24 / 18.63 ± 1.85

36.38 ± 0.86 / 52.08 ± 0.66

28.33 ± 2.31 / 45.29 ± 1.78

12.6.1

robinsmits/Qwen1.5-7B-Dutch-Chat-Sft-Bf16 (few-shot)

7719

152

32768

False

2,413 ± 463 / 700 ± 220

2.67

56.83 ± 2.31 / 46.81 ± 2.87

14.79 ± 1.96 / 41.48 ± 1.53

23.58 ± 2.69 / 50.85 ± 3.74

55.90 ± 1.80 / 70.07 ± 0.77

65.06 ± 1.20 / 18.45 ± 1.09

33.73 ± 0.75 / 50.05 ± 0.57

29.29 ± 1.83 / 45.90 ± 1.77

12.6.1

ReBatch/Reynaerde-7B-Instruct (few-shot)

7248

33

32768

False

2,562 ± 487 / 782 ± 247

2.72

59.16 ± 2.29 / 42.33 ± 2.15

10.39 ± 1.44 / 28.74 ± 1.05

19.50 ± 1.96 / 55.52 ± 3.92

60.96 ± 1.24 / 72.79 ± 0.95

68.19 ± 0.63 / 20.85 ± 0.78

32.93 ± 0.63 / 49.37 ± 0.55

28.04 ± 1.54 / 45.48 ± 1.35

13.0.0

robinsmits/Qwen1.5-7B-Dutch-Chat (few-shot)

7719

152

32768

False

4,686 ± 1,131 / 996 ± 326

2.72

57.81 ± 2.68 / 47.15 ± 2.77

14.62 ± 2.25 / 41.08 ± 1.81

25.34 ± 2.37 / 54.46 ± 3.43

56.81 ± 1.44 / 70.49 ± 0.68

60.51 ± 2.40 / 15.62 ± 1.65

34.78 ± 0.80 / 50.83 ± 0.58

30.29 ± 1.78 / 46.74 ± 1.68

12.5.3

ReBatch/Reynaerde-7B-Chat (few-shot)

7248

33

32768

False

2,554 ± 483 / 781 ± 247

2.75

56.22 ± 2.46 / 38.04 ± 1.69

11.22 ± 1.85 / 30.99 ± 1.36

20.04 ± 1.67 / 55.38 ± 3.62

61.15 ± 1.01 / 72.89 ± 0.88

68.28 ± 0.61 / 20.97 ± 0.82

32.44 ± 0.61 / 49.05 ± 0.48

31.24 ± 1.92 / 47.72 ± 1.69

13.0.0

ibm-granite/granite-3.0-8b-instruct (few-shot)

8171

49

4096

True

5,090 ± 937 / 1,423 ± 466

2.76

53.62 ± 2.29 / 40.51 ± 1.41

13.37 ± 1.25 / 36.94 ± 1.81

23.47 ± 1.79 / 60.17 ± 1.23

61.20 ± 1.02 / 72.98 ± 0.62

62.34 ± 3.24 / 18.49 ± 1.19

34.69 ± 0.59 / 50.59 ± 0.56

31.36 ± 1.90 / 46.78 ± 1.51

13.0.0

google/gemma-7b (few-shot)

8538

256

8192

True

1,378 ± 260 / 387 ± 119

2.80

47.75 ± 2.33 / 35.64 ± 1.89

7.68 ± 0.61 / 26.25 ± 1.18

28.28 ± 2.48 / 62.81 ± 1.70

61.49 ± 1.15 / 73.19 ± 0.81

65.60 ± 1.18 / 19.95 ± 1.23

42.19 ± 0.97 / 55.52 ± 0.93

31.45 ± 2.79 / 44.15 ± 3.09

12.9.1

12.10.0

ReBatch/Llama-3-8B-dutch (few-shot)

8030

128

8317

False

3,800 ± 1,275 / 566 ± 194

2.81

60.14 ± 2.00 / 44.91 ± 2.19

11.07 ± 1.98 / 34.77 ± 1.31

15.67 ± 3.75 / 40.14 ± 2.65

59.93 ± 1.17 / 71.20 ± 1.30

64.49 ± 1.40 / 18.40 ± 1.70

36.13 ± 1.24 / 51.76 ± 0.92

28.25 ± 2.23 / 44.23 ± 2.03

12.7.0

Rijgersberg/Mistral-7B-v0.1-chat-nl (few-shot)

7242

32

32768

False

5,907 ± 1,028 / 1,695 ± 549

2.86

56.73 ± 1.95 / 38.97 ± 1.84

11.08 ± 1.46 / 32.20 ± 1.43

19.41 ± 2.55 / 57.17 ± 2.38

58.91 ± 0.92 / 71.22 ± 0.72

68.05 ± 0.39 / 21.26 ± 0.65

31.39 ± 0.86 / 48.09 ± 0.72

16.49 ± 2.14 / 36.67 ± 1.72

12.5.2

mistralai/Mistral-7B-Instruct-v0.2 (few-shot)

7242

32

32768

False

634 ± 179 / 110 ± 35

2.87

55.56 ± 2.66 / 39.56 ± 2.13

12.37 ± 1.64 / 37.37 ± 1.35

21.50 ± 1.70 / 59.10 ± 1.32

50.77 ± 0.95 / 66.54 ± 0.79

67.99 ± 0.49 / 19.54 ± 0.55

22.86 ± 1.89 / 41.71 ± 1.45

24.80 ± 1.77 / 42.93 ± 1.38

9.3.1

9.2.0

9.3.1

12.4.0

9.3.2

mistralai/Mistral-7B-v0.1 (few-shot)

7242

32

32768

True

2,657 ± 524 / 880 ± 278

2.87

58.15 ± 1.14 / 40.78 ± 1.91

7.94 ± 1.25 / 31.02 ± 3.45

25.41 ± 3.46 / 61.11 ± 2.36

62.56 ± 1.10 / 73.16 ± 0.93

64.24 ± 0.91 / 17.54 ± 1.10

35.49 ± 0.57 / 51.51 ± 0.42

19.88 ± 1.80 / 39.13 ± 1.56

9.1.2

12.5.1

11.0.0

9.2.0

alpindale/Mistral-7B-v0.2-hf (few-shot)

7242

32

32768

True

1,841 ± 297 / 651 ± 193

2.90

56.76 ± 1.52 / 42.03 ± 1.98

7.11 ± 1.17 / 26.36 ± 2.97

23.55 ± 2.76 / 59.14 ± 3.18

61.89 ± 1.10 / 72.41 ± 1.08

64.33 ± 0.72 / 17.66 ± 0.87

34.92 ± 1.07 / 51.02 ± 0.85

23.87 ± 1.80 / 42.03 ± 1.58

12.5.2

ibm-granite/granite-3.0-2b-base (few-shot)

2534

49

4224

True

10,187 ± 2,363 / 2,204 ± 737

2.91

47.28 ± 1.57 / 36.12 ± 1.72

12.12 ± 1.92 / 35.44 ± 1.80

12.74 ± 2.68 / 52.69 ± 2.88

60.36 ± 1.36 / 71.20 ± 0.77

64.26 ± 0.75 / 16.55 ± 0.76

26.88 ± 0.86 / 44.76 ± 0.70

34.05 ± 1.57 / 50.08 ± 1.18

13.0.0

mistralai/Mistral-7B-v0.3 (few-shot)

7248

33

32768

True

4,120 ± 976 / 926 ± 306

2.91

56.52 ± 1.42 / 41.84 ± 1.84

7.02 ± 1.21 / 26.40 ± 2.96

23.41 ± 2.91 / 59.14 ± 3.11

61.90 ± 1.07 / 72.49 ± 1.05

64.37 ± 0.74 / 17.69 ± 0.90

34.93 ± 1.11 / 51.02 ± 0.87

23.73 ± 1.97 / 41.93 ± 1.68

12.10.4

12.10.5

12.10.4

ibm-granite/granite-3.0-2b-instruct (few-shot)

2634

49

4224

True

10,194 ± 2,403 / 2,193 ± 731

2.92

52.52 ± 1.62 / 44.69 ± 2.23

13.85 ± 1.90 / 36.43 ± 2.11

17.72 ± 1.86 / 57.31 ± 1.52

53.50 ± 1.16 / 67.02 ± 0.75

66.23 ± 0.48 / 18.18 ± 0.68

24.62 ± 0.65 / 43.32 ± 0.48

15.79 ± 1.37 / 35.56 ± 1.11

13.0.0

BramVanroy/GEITje-7B-ultra (few-shot)

7242

32

8192

False

2,475 ± 460 / 765 ± 238

2.95

42.20 ± 2.20 / 27.85 ± 1.11

12.78 ± 2.52 / 42.17 ± 1.91

18.23 ± 1.91 / 50.04 ± 2.54

53.41 ± 1.11 / 66.45 ± 0.46

68.30 ± 0.40 / 20.79 ± 0.63

26.92 ± 1.09 / 44.44 ± 0.85

25.72 ± 1.01 / 43.74 ± 0.69

10.0.1

12.4.0

10.0.1

neuralmagic/Sparse-Llama-3.1-8B-2of4 (few-shot)

8030

128

131072

True

2,841 ± 509 / 865 ± 263

2.97

34.21 ± 3.16 / 33.46 ± 2.25

11.94 ± 2.89 / 34.06 ± 1.51

6.18 ± 3.30 / 37.82 ± 4.01

60.44 ± 0.63 / 71.27 ± 0.78

64.48 ± 0.77 / 20.21 ± 0.93

29.76 ± 0.98 / 46.99 ± 0.77

38.79 ± 3.59 / 51.86 ± 3.32

13.2.0

microsoft/Phi-3-mini-4k-instruct (few-shot)

3821

32

4096

True

5,224 ± 1,371 / 1,063 ± 358

2.99

50.31 ± 1.94 / 41.54 ± 2.19

12.58 ± 1.62 / 36.56 ± 1.79

14.72 ± 1.84 / 50.23 ± 3.10

56.19 ± 0.80 / 66.72 ± 0.92

64.40 ± 0.77 / 17.47 ± 0.56

28.31 ± 0.70 / 46.18 ± 0.50

21.25 ± 1.25 / 40.78 ± 1.02

12.10.5

meta-llama/Llama-3.2-3B-Instruct (few-shot)

3213

128

131200

False

10,424 ± 2,641 / 2,081 ± 666

3.00

43.66 ± 2.01 / 40.23 ± 1.75

12.87 ± 1.32 / 37.23 ± 1.55

17.94 ± 3.48 / 55.88 ± 3.97

47.77 ± 1.82 / 64.44 ± 1.35

66.74 ± 1.00 / 18.33 ± 1.28

33.80 ± 0.84 / 50.30 ± 0.61

21.02 ± 1.29 / 40.27 ± 1.09

13.0.0

Rijgersberg/GEITje-7B-chat-v2 (few-shot)

7242

32

32768

False

5,908 ± 1,022 / 1,694 ± 551

3.03

42.12 ± 4.00 / 31.12 ± 1.86

11.06 ± 2.30 / 40.32 ± 1.64

19.71 ± 3.65 / 49.65 ± 4.28

59.19 ± 0.91 / 70.06 ± 0.82

65.55 ± 0.49 / 19.18 ± 0.67

27.71 ± 0.91 / 45.01 ± 0.71

18.03 ± 1.77 / 36.82 ± 1.76

12.5.2

google/gemma-2-2b-it (few-shot)

2614

256

8320

True

5,374 ± 1,233 / 1,193 ± 377

3.03

40.58 ± 2.08 / 28.95 ± 1.56

11.17 ± 1.32 / 32.96 ± 1.96

19.63 ± 2.61 / 52.65 ± 2.73

49.30 ± 1.25 / 67.27 ± 0.73

66.83 ± 0.86 / 19.21 ± 0.95

29.30 ± 1.02 / 46.88 ± 0.79

31.43 ± 1.42 / 47.87 ± 1.30

13.0.0

occiglot/occiglot-7b-eu5-instruct (few-shot)

7242

32

32768

False

2,088 ± 352 / 706 ± 214

3.03

53.78 ± 1.86 / 41.29 ± 2.07

7.78 ± 1.43 / 24.33 ± 1.57

16.23 ± 2.49 / 55.09 ± 3.18

63.09 ± 1.18 / 73.88 ± 0.72

66.46 ± 0.67 / 19.49 ± 0.92

28.37 ± 0.99 / 45.81 ± 0.88

15.25 ± 1.71 / 35.83 ± 1.42

12.5.2

12.2.0

12.3.1

12.4.0

12.3.1

meta-llama/Llama-2-13b-chat-hf (few-shot)

13016

32

4096

True

2,849 ± 622 / 723 ± 229

3.05

57.80 ± 1.53 / 39.43 ± 1.53

8.57 ± 1.27 / 29.84 ± 2.35

17.40 ± 1.54 / 57.26 ± 1.96

56.35 ± 0.85 / 69.69 ± 0.76

66.11 ± 0.58 / 18.31 ± 0.61

27.15 ± 1.09 / 44.95 ± 0.78

19.18 ± 1.46 / 38.84 ± 1.26

12.11.0

12.10.4

12.11.0

12.10.4

ibm-granite/granite-8b-code-base-4k (few-shot)

8055

49

4096

True

2,313 ± 423 / 682 ± 210

3.06

63.29 ± 2.51 / 52.18 ± 4.31

13.81 ± 1.66 / 36.99 ± 2.58

8.16 ± 1.97 / 44.29 ± 4.25

56.64 ± 0.68 / 66.29 ± 0.71

63.08 ± 0.90 / 15.69 ± 0.65

16.86 ± 1.16 / 37.54 ± 0.93

6.24 ± 0.78 / 29.25 ± 0.61

13.0.0

RuterNorway/Llama-2-13b-chat-norwegian (few-shot)

unknown

32

4096

False

3,254 ± 1,068 / 484 ± 173

3.07

57.66 ± 1.29 / 43.77 ± 2.78

8.41 ± 1.47 / 25.59 ± 1.30

16.93 ± 2.60 / 55.72 ± 3.35

56.29 ± 1.11 / 68.94 ± 0.81

66.22 ± 0.50 / 19.03 ± 0.51

25.70 ± 1.05 / 44.15 ± 0.82

17.92 ± 1.69 / 37.64 ± 1.43

9.3.1

microsoft/Phi-3-mini-128k-instruct (few-shot)

3821

32

131072

True

7,312 ± 1,668 / 1,609 ± 525

3.07

44.27 ± 2.14 / 34.47 ± 2.48

12.84 ± 1.82 / 36.64 ± 1.13

10.44 ± 1.58 / 48.93 ± 3.41

56.40 ± 1.14 / 68.02 ± 0.79

62.60 ± 0.62 / 15.63 ± 0.45

28.63 ± 0.84 / 46.39 ± 0.61

22.58 ± 0.97 / 41.78 ± 0.71

12.9.1

12.10.0

meta-llama/Llama-2-13b-hf (few-shot)

13016

32

4096

True

2,898 ± 637 / 736 ± 236

3.08

52.55 ± 1.64 / 43.32 ± 1.70

4.26 ± 2.09 / 28.32 ± 2.68

24.57 ± 3.54 / 54.94 ± 5.33

60.99 ± 0.95 / 72.74 ± 0.78

64.02 ± 0.84 / 16.97 ± 0.98

28.73 ± 0.90 / 45.62 ± 0.64

19.43 ± 1.53 / 38.57 ± 1.39

12.10.5

12.10.4

12.10.5

12.10.4

BramVanroy/fietje-2b-instruct (few-shot)

2775

50

2048

False

4,710 ± 1,040 / 1,188 ± 383

3.09

36.50 ± 2.72 / 28.73 ± 1.41

13.70 ± 2.29 / 40.77 ± 1.77

4.81 ± 2.20 / 43.19 ± 1.45

60.63 ± 0.62 / 71.62 ± 0.59

66.01 ± 0.42 / 17.97 ± 0.65

26.76 ± 1.00 / 44.94 ± 0.74

12.93 ± 1.54 / 34.12 ± 1.22

12.6.1

Rijgersberg/GEITje-7B (few-shot)

7242

32

32768

True

5,887 ± 1,087 / 1,600 ± 522

3.09

47.53 ± 1.90 / 32.42 ± 1.99

4.36 ± 2.96 / 28.11 ± 4.71

30.67 ± 4.45 / 63.78 ± 2.80

56.55 ± 0.70 / 67.56 ± 0.60

67.58 ± 0.85 / 22.14 ± 1.21

28.12 ± 0.98 / 44.50 ± 0.93

11.70 ± 2.42 / 32.05 ± 1.90

9.3.1

google/gemma-7b-it (few-shot)

8538

256

8317

False

1,792 ± 249 / 668 ± 203

3.10

53.93 ± 2.71 / 47.48 ± 2.09

12.83 ± 2.37 / 34.00 ± 2.26

6.58 ± 3.36 / 48.51 ± 3.35

53.45 ± 2.52 / 67.47 ± 0.88

65.89 ± 0.38 / 17.99 ± 0.46

19.58 ± 1.24 / 38.76 ± 1.06

13.68 ± 1.08 / 32.30 ± 0.99

12.10.0

BramVanroy/fietje-2b-chat (few-shot)

2775

50

2048

False

4,704 ± 1,015 / 1,185 ± 375

3.12

39.57 ± 2.74 / 31.81 ± 1.55

13.25 ± 2.12 / 39.92 ± 1.66

9.31 ± 1.92 / 50.99 ± 2.59

60.26 ± 0.62 / 71.03 ± 0.65

65.37 ± 0.53 / 17.63 ± 0.74

26.63 ± 1.33 / 44.86 ± 1.00

11.62 ± 1.28 / 32.71 ± 1.00

12.6.1

Rijgersberg/GEITje-7B-chat (few-shot)

7242

32

32768

False

5,920 ± 1,028 / 1,696 ± 550

3.15

50.69 ± 1.67 / 35.96 ± 2.63

8.16 ± 1.68 / 27.37 ± 1.95

20.45 ± 2.12 / 59.00 ± 1.21

54.48 ± 0.86 / 66.71 ± 0.59

66.92 ± 0.50 / 20.19 ± 0.69

24.89 ± 1.55 / 42.34 ± 1.14

9.84 ± 1.87 / 30.90 ± 1.58

12.5.2

meta-llama/Llama-2-7b-chat-hf (few-shot)

6738

32

4096

False

2,643 ± 455 / 800 ± 247

3.15

50.23 ± 2.34 / 37.12 ± 3.30

10.07 ± 1.84 / 35.66 ± 2.24

14.73 ± 1.62 / 54.59 ± 2.24

53.42 ± 0.80 / 66.24 ± 0.84

67.59 ± 0.56 / 18.45 ± 0.65

20.19 ± 1.26 / 39.24 ± 1.03

11.42 ± 1.29 / 32.50 ± 1.10

9.3.1

12.4.0

9.3.1

occiglot/occiglot-7b-eu5 (few-shot)

7242

32

32768

True

2,219 ± 427 / 717 ± 224

3.15

51.31 ± 2.32 / 42.95 ± 2.58

7.41 ± 1.24 / 26.93 ± 1.56

13.04 ± 1.93 / 53.54 ± 2.70

59.28 ± 1.15 / 69.67 ± 0.95

64.66 ± 0.74 / 18.22 ± 0.85

27.12 ± 0.86 / 44.36 ± 0.75

13.99 ± 2.04 / 34.45 ± 1.89

12.5.2

12.1.0

12.2.0

Qwen/Qwen1.5-4B-Chat (few-shot)

3950

152

32768

False

4,347 ± 893 / 1,135 ± 365

3.16

42.52 ± 2.25 / 37.46 ± 3.08

14.68 ± 1.40 / 40.53 ± 1.64

4.07 ± 2.16 / 35.24 ± 1.77

55.18 ± 0.74 / 66.50 ± 0.80

62.75 ± 0.84 / 14.83 ± 0.70

24.25 ± 1.22 / 43.08 ± 0.87

16.21 ± 1.51 / 37.11 ± 1.16

12.5.2

10.0.1

12.1.0

12.5.2

12.1.0

ibm-granite/granite-3.0-3b-a800m-instruct (few-shot)

3374

49

4096

True

10,246 ± 3,021 / 1,629 ± 550

3.16

49.25 ± 2.57 / 36.48 ± 2.14

9.45 ± 1.76 / 39.66 ± 1.14

11.87 ± 2.68 / 47.32 ± 3.85

54.20 ± 1.44 / 67.04 ± 0.75

64.77 ± 0.96 / 16.74 ± 0.86

21.52 ± 1.18 / 40.76 ± 0.95

22.32 ± 2.02 / 39.89 ± 1.82

13.0.0

mistralai/Mistral-7B-Instruct-v0.1 (few-shot)

7242

32

32768

False

634 ± 179 / 110 ± 35

3.16

52.72 ± 2.58 / 33.51 ± 1.22

7.91 ± 2.16 / 27.82 ± 1.97

18.14 ± 2.10 / 55.42 ± 3.05

52.75 ± 0.88 / 67.15 ± 1.08

64.77 ± 0.97 / 16.55 ± 0.81

26.06 ± 0.77 / 44.08 ± 0.51

14.26 ± 1.48 / 35.14 ± 1.18

9.3.1

12.4.0

9.3.1

ibm-granite/granite-8b-code-instruct-4k (few-shot)

8055

49

4096

True

5,617 ± 995 / 1,623 ± 540

3.19

60.72 ± 2.14 / 45.52 ± 2.46

12.38 ± 1.62 / 29.91 ± 1.91

10.96 ± 1.47 / 47.97 ± 3.45

51.20 ± 0.91 / 61.75 ± 0.63

63.23 ± 0.96 / 16.22 ± 0.74

15.26 ± 0.67 / 35.76 ± 0.34

4.03 ± 0.90 / 27.17 ± 0.73

13.0.0

BramVanroy/GEITje-7B-ultra-sft (few-shot)

7242

32

8192

False

5,979 ± 1,044 / 1,724 ± 559

3.23

39.41 ± 2.93 / 30.59 ± 1.59

7.00 ± 3.04 / 35.01 ± 3.72

16.10 ± 2.34 / 52.05 ± 3.60

53.02 ± 0.97 / 65.63 ± 0.72

67.74 ± 0.43 / 20.32 ± 0.57

25.80 ± 0.75 / 42.85 ± 0.76

12.49 ± 2.15 / 33.04 ± 2.08

12.5.2

01-ai/Yi-1.5-6B (few-shot)

6061

64

4224

True

2,867 ± 550 / 793 ± 253

3.26

51.18 ± 1.62 / 35.45 ± 1.88

9.23 ± 2.84 / 19.38 ± 4.37

1.99 ± 2.56 / 34.69 ± 1.59

54.66 ± 1.25 / 65.31 ± 1.06

60.81 ± 1.16 / 15.41 ± 0.63

28.03 ± 1.02 / 45.90 ± 0.79

23.64 ± 1.21 / 42.59 ± 0.96

13.0.0

BramVanroy/fietje-2b (few-shot)

2780

51

2048

True

4,804 ± 1,045 / 1,220 ± 392

3.32

33.92 ± 3.43 / 28.63 ± 2.42

13.39 ± 1.64 / 41.03 ± 1.87

6.75 ± 2.55 / 41.28 ± 2.37

58.57 ± 1.03 / 69.39 ± 0.79

61.49 ± 0.90 / 16.13 ± 0.80

24.66 ± 0.67 / 42.68 ± 0.55

4.78 ± 1.80 / 27.19 ± 1.20

12.6.1

ibm-granite/granite-3.0-3b-a800m-base (few-shot)

3374

49

4096

True

10,504 ± 3,028 / 1,678 ± 559

3.32

42.52 ± 3.31 / 33.08 ± 2.70

9.91 ± 1.71 / 35.24 ± 2.62

0.69 ± 2.82 / 36.10 ± 2.58

56.95 ± 1.18 / 66.87 ± 1.37

63.71 ± 0.60 / 17.04 ± 0.70

20.93 ± 1.42 / 40.35 ± 1.23

24.42 ± 1.73 / 42.26 ± 1.49

13.0.0

timpal0l/Mistral-7B-v0.1-flashback-v2 (few-shot)

7242

32

32768

True

5,054 ± 1,200 / 1,056 ± 339

3.34

54.56 ± 2.96 / 37.86 ± 2.49

8.43 ± 1.27 / 24.23 ± 0.94

10.99 ± 2.55 / 50.46 ± 4.17

55.91 ± 1.08 / 66.78 ± 1.13

57.88 ± 1.41 / 12.36 ± 1.30

25.12 ± 1.17 / 43.50 ± 0.92

10.65 ± 1.55 / 31.51 ± 1.52

12.5.3

01-ai/Yi-6B (few-shot)

6061

64

4096

True

6,435 ± 1,316 / 1,632 ± 549

3.35

46.34 ± 2.00 / 33.30 ± 1.78

8.96 ± 1.44 / 18.10 ± 2.39

0.88 ± 1.23 / 33.53 ± 0.48

55.33 ± 1.28 / 66.50 ± 0.94

58.35 ± 1.57 / 14.29 ± 0.85

29.33 ± 0.91 / 46.94 ± 0.68

20.27 ± 1.38 / 39.32 ± 1.12

9.3.2

10.0.0

12.5.1

12.0.0

10.0.1

meta-llama/Llama-3.2-3B (few-shot)

3213

128

131200

True

3,713 ± 877 / 836 ± 267

3.35

47.40 ± 3.29 / 33.11 ± 2.04

7.90 ± 1.98 / 30.71 ± 1.89

3.10 ± 1.93 / 34.24 ± 0.73

56.53 ± 1.48 / 68.47 ± 1.35

62.27 ± 2.04 / 16.54 ± 1.58

29.18 ± 0.87 / 46.04 ± 0.54

10.34 ± 1.08 / 32.08 ± 1.06

13.0.0

meta-llama/Llama-2-7b-hf (few-shot)

6738

32

4096

True

930 ± 310 / 128 ± 43

3.38

40.49 ± 4.32 / 30.86 ± 2.27

7.10 ± 1.85 / 27.42 ± 1.76

18.66 ± 2.39 / 55.25 ± 3.77

59.92 ± 0.61 / 70.24 ± 0.75

62.58 ± 1.13 / 16.33 ± 0.81

17.36 ± 1.12 / 37.44 ± 0.99

6.68 ± 1.82 / 29.30 ± 1.02

9.2.0

12.5.1

11.0.0

9.2.0

Qwen/Qwen1.5-4B (few-shot)

3950

152

32768

True

3,248 ± 739 / 761 ± 252

3.44

35.74 ± 3.22 / 31.74 ± 2.24

12.55 ± 1.39 / 39.80 ± 1.38

0.23 ± 0.44 / 33.35 ± 0.31

51.30 ± 1.63 / 64.17 ± 0.87

58.90 ± 1.59 / 14.17 ± 0.80

22.20 ± 1.53 / 41.49 ± 1.08

13.11 ± 1.85 / 33.86 ± 1.69

12.5.2

10.0.1

12.1.0

TrustLLMeu/baseline-7-8b_1t-tokens_llama (few-shot)

7800

100

4096

True

6,197 ± 1,118 / 1,730 ± 577

3.47

48.24 ± 2.28 / 40.24 ± 1.90

11.37 ± 1.34 / 33.54 ± 1.83

10.73 ± 1.76 / 48.26 ± 4.65

54.83 ± 1.16 / 65.28 ± 1.11

64.32 ± 0.90 / 18.12 ± 1.06

0.72 ± 1.25 / 24.90 ± 0.92

-0.47 ± 0.87 / 24.28 ± 0.68

13.0.0

AI-Sweden-Models/gpt-sw3-20b (few-shot)

20918

64

2048

True

1,875 ± 673 / 261 ± 91

3.48

35.30 ± 3.76 / 33.68 ± 1.80

15.67 ± 2.21 / 31.30 ± 4.51

1.76 ± 2.37 / 47.60 ± 1.68

45.05 ± 1.68 / 55.38 ± 1.66

59.15 ± 1.54 / 14.60 ± 0.72

6.24 ± 1.54 / 29.02 ± 1.30

0.47 ± 1.20 / 24.89 ± 0.59

9.3.1

allenai/OLMo-1.7-7B-hf (few-shot)

6888

50

4096

True

3,371 ± 876 / 561 ± 184

3.56

46.95 ± 2.32 / 36.13 ± 1.88

4.34 ± 2.10 / 19.37 ± 2.08

3.46 ± 1.91 / 41.32 ± 3.08

57.07 ± 0.89 / 68.14 ± 0.65

61.31 ± 0.64 / 17.24 ± 0.72

18.21 ± 0.70 / 37.51 ± 0.63

9.03 ± 1.04 / 29.35 ± 0.77

12.10.5

12.10.4

12.10.5

12.10.4

google/gemma-2-2b (few-shot)

2614

256

8320

True

5,235 ± 1,226 / 1,154 ± 366

3.61

22.63 ± 4.98 / 22.71 ± 2.86

8.11 ± 1.55 / 28.07 ± 1.80

8.04 ± 1.79 / 48.95 ± 2.97

52.39 ± 2.14 / 65.22 ± 1.11

61.90 ± 1.04 / 17.09 ± 0.93

22.76 ± 0.76 / 41.67 ± 0.80

8.50 ± 1.32 / 30.16 ± 1.11

13.0.0

ibm-granite/granite-3b-code-instruct-2k (few-shot)

3483

49

2048

True

9,059 ± 1,947 / 2,201 ± 728

3.61

48.53 ± 3.89 / 38.20 ± 2.92

10.15 ± 1.55 / 22.01 ± 1.44

4.88 ± 2.27 / 38.78 ± 3.56

45.38 ± 0.93 / 56.09 ± 1.05

59.56 ± 1.09 / 15.20 ± 0.56

7.38 ± 1.06 / 29.60 ± 0.71

2.69 ± 0.56 / 25.85 ± 0.33

13.0.0

MaLA-LM/emma-500-llama2-7b (few-shot)

6738

32

4096

True

6,275 ± 1,193 / 1,755 ± 578

3.62

36.61 ± 3.37 / 31.91 ± 2.20

8.77 ± 1.80 / 25.31 ± 1.42

3.52 ± 2.07 / 35.34 ± 1.61

59.51 ± 0.97 / 70.33 ± 0.64

54.50 ± 5.78 / 15.01 ± 1.59

14.90 ± 1.44 / 35.20 ± 1.09

7.26 ± 1.56 / 29.91 ± 1.18

13.0.0

ibm-granite/granite-3b-code-base-2k (few-shot)

3483

49

2048

True

2,732 ± 868 / 662 ± 238

3.64

50.88 ± 3.41 / 39.51 ± 3.29

12.39 ± 1.50 / 30.05 ± 1.80

3.31 ± 1.19 / 37.97 ± 4.40

48.44 ± 1.06 / 58.85 ± 0.75

52.50 ± 2.95 / 10.11 ± 2.12

9.22 ± 0.79 / 31.06 ± 0.75

4.45 ± 1.38 / 28.26 ± 1.08

13.0.0

ibm-granite/granite-7b-instruct (few-shot)

6738

32

4096

True

3,136 ± 558 / 942 ± 290

3.66

33.73 ± 2.02 / 30.41 ± 1.57

7.45 ± 1.77 / 22.28 ± 1.35

3.78 ± 2.04 / 50.30 ± 1.51

43.60 ± 1.46 / 58.94 ± 1.08

64.37 ± 0.44 / 16.25 ± 0.30

13.67 ± 1.39 / 34.95 ± 1.02

11.51 ± 0.68 / 33.35 ± 0.60

13.2.0

13.0.0

13.2.0

AI-Sweden-Models/gpt-sw3-20b-instruct (few-shot)

20918

64

2048

True

1,831 ± 587 / 268 ± 90

3.68

24.44 ± 1.62 / 25.02 ± 1.72

18.40 ± 2.14 / 40.21 ± 2.51

4.85 ± 2.01 / 49.10 ± 2.56

39.83 ± 1.08 / 52.69 ± 1.15

54.89 ± 1.27 / 13.01 ± 0.47

4.38 ± 1.45 / 27.07 ± 1.30

1.95 ± 0.83 / 24.88 ± 0.71

9.3.1

ibm-granite/granite-7b-base (few-shot)

6738

32

2048

True

4,405 ± 1,098 / 1,032 ± 345

3.70

37.39 ± 3.37 / 32.77 ± 1.97

7.51 ± 1.57 / 19.22 ± 1.72

3.11 ± 0.88 / 50.54 ± 0.90

49.60 ± 1.28 / 61.06 ± 0.94

57.63 ± 1.86 / 12.27 ± 0.83

14.52 ± 1.25 / 35.34 ± 0.95

6.60 ± 1.13 / 29.30 ± 0.97

12.10.5

12.10.8

meta-llama/Llama-3.2-1B-Instruct (few-shot)

1236

128

131200

False

7,436 ± 1,846 / 1,508 ± 479

3.70

42.01 ± 2.06 / 37.16 ± 1.98

9.15 ± 1.70 / 32.55 ± 2.69

1.11 ± 2.15 / 36.71 ± 3.89

40.04 ± 1.61 / 53.75 ± 1.10

58.72 ± 1.05 / 12.47 ± 0.62

17.71 ± 0.69 / 38.01 ± 0.61

6.98 ± 1.11 / 28.05 ± 1.00

13.0.0

RuterNorway/Llama-2-7b-chat-norwegian (few-shot)

unknown

32

4096

False

10,890 ± 2,686 / 2,186 ± 750

3.78

35.49 ± 3.10 / 29.35 ± 2.75

11.36 ± 1.56 / 30.66 ± 3.68

2.52 ± 2.14 / 42.60 ± 4.80

37.49 ± 1.37 / 47.34 ± 1.53

62.24 ± 0.86 / 15.62 ± 0.60

5.41 ± 1.15 / 27.75 ± 0.75

0.15 ± 0.98 / 25.59 ± 0.57

9.3.1

12.5.2

9.3.1

utter-project/EuroLLM-1.7B-Instruct (few-shot)

1657

128

4096

True

15,009 ± 4,072 / 2,702 ± 878

3.79

32.45 ± 2.17 / 30.83 ± 2.31

7.03 ± 2.08 / 34.16 ± 1.83

5.58 ± 1.32 / 44.79 ± 3.28

51.18 ± 1.19 / 61.82 ± 1.19

61.96 ± 0.80 / 14.66 ± 0.48

2.53 ± 1.10 / 25.95 ± 1.10

1.04 ± 0.74 / 25.13 ± 0.68

13.1.0

ibm-granite/granite-3.0-1b-a400m-instruct (few-shot)

1335

49

4096

True

7,964 ± 2,255 / 1,299 ± 433

3.80

30.60 ± 2.69 / 26.10 ± 1.90

13.26 ± 1.54 / 37.96 ± 2.44

1.04 ± 1.52 / 45.97 ± 3.20

39.69 ± 1.18 / 50.12 ± 0.89

61.20 ± 1.00 / 13.03 ± 0.49

2.10 ± 1.04 / 26.02 ± 0.92

-0.52 ± 0.72 / 23.98 ± 0.59

13.2.0

google/gemma-2b-it (few-shot)

2506

256

8192

False

6,471 ± 1,142 / 1,961 ± 584

3.87

38.85 ± 3.77 / 32.18 ± 2.49

11.25 ± 1.90 / 28.36 ± 1.81

-2.27 ± 1.37 / 37.91 ± 2.26

45.95 ± 1.11 / 56.54 ± 0.95

51.99 ± 8.25 / 12.75 ± 2.04

10.97 ± 0.92 / 32.28 ± 0.76

1.45 ± 1.15 / 24.79 ± 0.52

12.5.2

12.1.0

12.4.0

12.1.0

google/gemma-2b (few-shot)

2506

256

8192

True

6,087 ± 1,046 / 1,902 ± 563

3.91

16.90 ± 4.91 / 17.38 ± 4.30

9.95 ± 0.78 / 27.94 ± 1.43

0.41 ± 1.03 / 33.54 ± 0.32

49.15 ± 1.55 / 59.16 ± 1.44

58.61 ± 2.22 / 13.67 ± 1.15

10.94 ± 0.50 / 31.87 ± 0.76

3.29 ± 0.95 / 26.85 ± 0.49

12.5.2

12.1.0

Qwen/Qwen1.5-1.8B-Chat (few-shot)

1837

152

32768

False

8,304 ± 1,846 / 1,933 ± 617

3.92

23.44 ± 5.09 / 25.00 ± 2.33

6.82 ± 1.82 / 30.97 ± 2.65

4.11 ± 1.73 / 43.70 ± 3.47

33.16 ± 1.61 / 46.66 ± 1.27

60.91 ± 0.99 / 12.65 ± 0.41

12.11 ± 0.90 / 33.62 ± 0.61

6.41 ± 1.13 / 29.73 ± 0.82

12.5.2

11.0.0

12.1.0

12.5.0

12.1.0

stabilityai/stablelm-2-1_6b (few-shot)

1645

100

4096

True

7,259 ± 2,120 / 1,240 ± 432

3.92

36.58 ± 3.88 / 33.82 ± 2.87

6.32 ± 1.30 / 24.04 ± 1.14

4.01 ± 2.01 / 36.03 ± 1.61

52.81 ± 0.81 / 63.87 ± 1.27

52.24 ± 1.79 / 10.76 ± 0.68

10.24 ± 1.06 / 31.02 ± 0.79

6.75 ± 0.83 / 28.50 ± 0.84

12.10.8

HuggingFaceTB/SmolLM2-1.7B-Instruct (few-shot)

1711

49

8192

True

15,971 ± 3,654 / 3,609 ± 1,197

3.95

31.84 ± 3.39 / 28.66 ± 1.77

1.56 ± 3.25 / 28.78 ± 2.60

5.05 ± 1.34 / 43.99 ± 4.14

40.55 ± 0.77 / 48.56 ± 0.95

60.35 ± 1.16 / 13.63 ± 0.63

13.39 ± 0.96 / 34.55 ± 0.65

4.57 ± 0.96 / 27.42 ± 0.73

13.1.0

Tweeties/tweety-7b-dutch-v24a (few-shot)

7391

50

1024

True

2,971 ± 423 / 1,351 ± 410

3.96

35.83 ± 3.06 / 29.15 ± 2.80

12.47 ± 2.51 / 40.41 ± 2.26

16.81 ± 3.31 / 53.38 ± 5.08

0.00 ± 0.00 / 14.18 ± 0.38

51.61 ± 0.65 / 9.48 ± 0.19

20.21 ± 1.63 / 37.93 ± 1.39

5.94 ± 1.54 / 28.98 ± 0.98

12.6.1

HuggingFaceTB/SmolLM2-1.7B (few-shot)

1711

49

8192

True

16,249 ± 3,690 / 3,689 ± 1,226

4.01

22.84 ± 5.42 / 25.11 ± 3.52

4.60 ± 2.12 / 29.94 ± 1.50

2.55 ± 1.41 / 40.88 ± 3.15

40.33 ± 1.19 / 48.35 ± 1.31

58.31 ± 1.50 / 13.08 ± 0.51

14.32 ± 0.71 / 35.65 ± 0.50

3.87 ± 1.02 / 27.14 ± 0.66

13.1.0

NorwAI/NorwAI-Mistral-7B (few-shot)

7537

68

4096

True

3,035 ± 503 / 911 ± 300

4.01

24.15 ± 5.73 / 26.49 ± 4.13

8.31 ± 1.56 / 20.06 ± 1.06

1.60 ± 1.71 / 41.51 ± 3.60

37.08 ± 1.76 / 49.32 ± 0.87

59.43 ± 1.17 / 14.57 ± 0.83

3.04 ± 1.28 / 26.56 ± 0.79

3.70 ± 1.54 / 26.98 ± 1.26

12.10.4

Qwen/Qwen1.5-0.5B-Chat (few-shot)

620

152

32768

False

11,740 ± 3,000 / 2,209 ± 721

4.09

18.66 ± 4.43 / 17.56 ± 4.28

8.59 ± 3.20 / 29.65 ± 5.10

0.34 ± 2.02 / 43.92 ± 3.15

26.74 ± 1.57 / 35.03 ± 2.14

62.36 ± 0.81 / 11.72 ± 0.73

2.09 ± 1.13 / 25.90 ± 0.60

1.44 ± 1.12 / 26.50 ± 0.68

12.5.2

11.0.0

12.1.0

12.5.0

12.1.0

LumiOpen/Viking-13B (few-shot)

14030

131

4224

True

840 ± 79 / 400 ± 124

4.12

36.74 ± 3.36 / 32.36 ± 1.39

8.57 ± 2.44 / 34.17 ± 2.59

3.01 ± 1.94 / 46.03 ± 4.19

32.32 ± 1.55 / 40.73 ± 1.64

51.76 ± 1.36 / 10.67 ± 0.56

0.67 ± 1.15 / 24.89 ± 0.79

-0.30 ± 1.01 / 23.52 ± 0.38

12.5.2

Qwen/Qwen1.5-1.8B (few-shot)

1837

152

32768

True

5,666 ± 1,328 / 1,256 ± 408

4.15

11.66 ± 6.46 / 15.15 ± 4.38

5.20 ± 1.78 / 35.43 ± 2.14

2.89 ± 1.91 / 41.36 ± 4.63

34.60 ± 2.17 / 48.83 ± 1.05

55.00 ± 1.73 / 11.21 ± 0.64

13.56 ± 0.64 / 34.17 ± 0.48

5.89 ± 0.82 / 29.17 ± 0.66

12.5.2

10.0.1

12.1.0

ibm-granite/granite-3.0-1b-a400m-base (few-shot)

1385

49

4096

True

7,808 ± 2,183 / 1,289 ± 428

4.17

12.76 ± 7.37 / 14.65 ± 5.86

9.35 ± 1.70 / 31.57 ± 6.23

0.69 ± 1.52 / 44.03 ± 3.52

37.71 ± 0.79 / 47.08 ± 1.12

56.45 ± 1.86 / 12.01 ± 0.44

-0.05 ± 0.51 / 24.48 ± 0.64

-0.85 ± 0.88 / 23.82 ± 0.73

13.2.0

meta-llama/Llama-3.2-1B (few-shot)

1236

128

131200

True

7,577 ± 1,884 / 1,555 ± 492

4.21

22.03 ± 4.43 / 19.22 ± 3.92

4.25 ± 2.95 / 26.57 ± 3.31

1.46 ± 1.83 / 42.29 ± 4.01

41.76 ± 1.92 / 52.60 ± 1.99

51.56 ± 1.45 / 10.34 ± 0.52

5.46 ± 1.05 / 28.09 ± 0.78

-0.58 ± 0.81 / 23.58 ± 0.50

13.0.0

Qwen/Qwen1.5-0.5B (few-shot)

620

152

32768

True

11,371 ± 2,924 / 2,122 ± 692

4.22

28.30 ± 3.90 / 28.67 ± 3.15

4.54 ± 2.76 / 26.53 ± 3.74

-0.42 ± 2.41 / 37.60 ± 3.89

20.81 ± 2.21 / 29.05 ± 2.31

58.40 ± 2.03 / 10.34 ± 0.54

7.44 ± 0.51 / 29.86 ± 0.35

1.70 ± 1.42 / 26.04 ± 0.91

12.5.2

10.0.1

12.1.0

allenai/OLMo-7B (few-shot)

6888

50

2176

True

5,403 ± 1,133 / 1,294 ± 423

4.24

37.37 ± 2.22 / 30.45 ± 2.45

9.55 ± 1.82 / 23.90 ± 1.53

0.05 ± 1.35 / 35.78 ± 2.30

34.81 ± 1.54 / 46.37 ± 1.51

45.22 ± 0.39 / 8.61 ± 0.17

3.92 ± 1.04 / 27.22 ± 0.63

0.16 ± 0.55 / 25.29 ± 0.61

12.5.2

NorwAI/NorwAI-Llama2-7B (few-shot)

7033

68

4096

True

4,438 ± 1,128 / 1,028 ± 346

4.30

22.50 ± 2.27 / 24.09 ± 2.40

6.04 ± 1.51 / 18.08 ± 2.09

-0.61 ± 1.30 / 46.51 ± 2.55

26.96 ± 1.55 / 35.73 ± 1.87

56.55 ± 3.05 / 13.22 ± 1.01

0.27 ± 1.29 / 24.94 ± 0.91

1.47 ± 0.79 / 25.75 ± 0.39

12.10.4

HuggingFaceTB/SmolLM2-360M (few-shot)

362

49

8192

True

22,023 ± 6,203 / 3,675 ± 1,231

4.34

20.95 ± 2.02 / 25.63 ± 1.96

6.84 ± 1.76 / 27.74 ± 5.49

-1.50 ± 1.30 / 34.07 ± 0.45

22.67 ± 1.77 / 30.14 ± 1.68

53.89 ± 2.61 / 10.53 ± 0.37

-0.45 ± 1.59 / 24.24 ± 0.79

-0.31 ± 1.39 / 23.58 ± 0.40

13.1.0

HuggingFaceTB/SmolLM2-360M-Instruct (few-shot)

362

49

8192

True

21,777 ± 6,115 / 3,617 ± 1,211

4.45

15.68 ± 5.54 / 22.21 ± 5.42

6.73 ± 2.20 / 27.67 ± 4.00

0.63 ± 1.05 / 43.48 ± 2.98

19.73 ± 1.42 / 27.47 ± 1.35

50.53 ± 2.03 / 9.14 ± 0.47

1.30 ± 0.75 / 24.43 ± 0.78

-0.36 ± 0.85 / 24.09 ± 0.49

13.1.0

state-spaces/mamba-2.8b-hf (few-shot)

2768

50

32896

True

2,722 ± 495 / 766 ± 250

4.52

22.95 ± 1.99 / 23.05 ± 1.55

2.40 ± 1.70 / 22.89 ± 4.81

3.12 ± 1.51 / 45.58 ± 4.56

22.40 ± 1.73 / 32.09 ± 1.56

50.36 ± 2.03 / 9.96 ± 0.95

-0.48 ± 0.81 / 24.57 ± 0.67

1.28 ± 1.03 / 26.62 ± 0.80

13.0.0

allenai/OLMo-7B-Twin-2T (few-shot)

6888

50

2176

True

5,484 ± 1,125 / 1,317 ± 425

4.56

18.70 ± 5.76 / 19.58 ± 4.59

3.70 ± 1.69 / 17.91 ± 1.48

2.19 ± 2.08 / 45.43 ± 3.44

38.08 ± 1.07 / 48.44 ± 1.55

43.79 ± 0.51 / 7.59 ± 0.20

0.65 ± 1.39 / 25.05 ± 1.02

-0.02 ± 0.92 / 24.75 ± 0.66

12.5.2

allenai/OLMo-1B (few-shot)

1177

50

2176

True

8,536 ± 1,926 / 1,940 ± 619

4.75

22.58 ± 5.05 / 26.82 ± 3.69

4.92 ± 2.71 / 19.51 ± 4.22

-1.27 ± 1.85 / 41.38 ± 3.59

6.64 ± 1.96 / 11.74 ± 1.62

43.43 ± 0.37 / 7.83 ± 0.14

-0.56 ± 1.27 / 24.38 ± 0.98

0.29 ± 1.06 / 25.10 ± 0.83

12.5.2

12.1.0

HuggingFaceTB/SmolLM2-135M-Instruct (few-shot)

135

49

8192

True

25,602 ± 7,583 / 3,953 ± 1,325

4.76

15.82 ± 3.13 / 16.46 ± 2.61

-0.62 ± 1.55 / 16.18 ± 1.88

1.16 ± 1.38 / 34.30 ± 1.27

3.25 ± 1.17 / 5.89 ± 0.97

54.82 ± 3.44 / 9.74 ± 0.58

1.36 ± 1.10 / 25.96 ± 0.81

0.34 ± 1.33 / 25.40 ± 0.83

13.1.0

HuggingFaceTB/SmolLM2-135M (few-shot)

135

49

8192

True

26,346 ± 7,812 / 4,082 ± 1,372

4.79

17.49 ± 2.94 / 18.59 ± 2.66

2.01 ± 1.88 / 15.88 ± 1.32

-0.02 ± 0.15 / 34.86 ± 2.12

0.53 ± 0.36 / 3.23 ± 0.51

52.46 ± 1.93 / 9.47 ± 0.33

0.33 ± 0.71 / 25.12 ± 0.63

-0.10 ± 0.85 / 24.87 ± 0.65

13.1.0

RJuro/kanelsnegl-v0.1 (few-shot)

7242

32

512

True

5,847 ± 1,029 / 1,640 ± 525

4.88

0.00 ± 0.00 / 0.00 ± 0.00

0.95 ± 1.17 / 9.87 ± 0.86

0.00 ± 0.00 / 33.34 ± 0.31

0.00 ± 0.00 / 5.43 ± 0.58

60.14 ± 0.15 / 10.07 ± 0.16

0.11 ± 0.64 / 24.32 ± 0.78

-0.13 ± 0.82 / 23.60 ± 0.33

9.3.1

12.5.1

9.3.1

ssmits/Falcon2-5.5B-multilingual (few-shot)

5465

65

8192

True

7,692 ± 1,423 / 1,960 ± 644

5.16

0.00 ± 0.00 / 0.00 ± 0.00

0.00 ± 0.00 / 8.62 ± 0.30

0.00 ± 0.00 / 33.34 ± 0.31

0.00 ± 0.00 / 0.01 ± 0.00

43.74 ± 0.57 / 0.22 ± 0.04

0.32 ± 0.67 / 24.79 ± 0.61

1.73 ± 1.07 / 26.61 ± 0.61

13.0.0

NorwAI/NorwAI-Mistral-7B-pretrain (few-shot)

7537

68

4096

True

3,024 ± 496 / 909 ± 301

5.17

3.80 ± 1.23 / 4.24 ± 1.19

0.97 ± 1.50 / 13.00 ± 2.52

-0.37 ± 0.55 / 33.40 ± 0.35

0.40 ± 0.25 / 2.98 ± 0.44

45.25 ± 4.22 / 6.15 ± 1.03

-0.98 ± 0.91 / 24.82 ± 0.88

-0.05 ± 1.07 / 24.51 ± 0.88

12.10.4

ai-forever/mGPT (few-shot)

unknown

100

1024

True

11,734 ± 3,124 / 2,174 ± 720

5.47

0.11 ± 0.21 / 0.27 ± 0.53

-0.67 ± 1.33 / 8.96 ± 0.37

-0.97 ± 1.56 / 34.83 ± 1.94

0.29 ± 0.21 / 1.56 ± 0.19

30.20 ± 0.68 / 2.14 ± 0.04

1.45 ± 0.84 / 24.91 ± 0.60

-0.56 ± 0.80 / 23.53 ± 0.49

9.3.1

10.0.1

11.0.0

12.5.1

12.0.0

11.0.0