{ "architectures": [ "Gemma3ForConditionalGeneration" ], "boi_token_index": 255999, "eoi_token_index": 256000, "eos_token_id": [ 1, 106 ], "image_token_index": 262144, "initializer_range": 0.02, "mm_tokens_per_image": 256, "model_type": "gemma3", "quantization_config": { "include_input_output_embeddings": true, "modules_to_not_convert": [], "quant_method": "torchao", "quant_type": { "default": { "_data": { "module_fqn_to_config": { "_default": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int4", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "group_size": 32 }, "_type": "PerGroup", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.language_model.embed_tokens": { "_data": { "granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "scale_dtype": null, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" } }, "_type": "IntxWeightOnlyConfig", "_version": 2 }, "model.vision_tower.vision_model.embeddings.position_embedding": { "_data": { "granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "scale_dtype": null, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" } }, "_type": "IntxWeightOnlyConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.0.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.1.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.10.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.11.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.12.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.13.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.14.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.15.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.16.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.17.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.18.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.19.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.2.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.20.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.21.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.22.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.23.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.24.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.25.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.26.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.3.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.4.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.5.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.6.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.7.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.8.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 }, "model.vision_tower.vision_model.encoder.layers.9.mlp.fc2": { "_data": { "act_mapping_type": { "_data": "ASYMMETRIC", "_type": "MappingType" }, "intx_choose_qparams_algorithm": "hqq_scale_only", "intx_packing_format": { "_data": "UNPACKED_TO_INT8", "_type": "IntxPackingFormat" }, "layout": { "_data": {}, "_type": "QDQLayout", "_version": 1 }, "weight_dtype": { "_data": "int8", "_type": "torch.dtype" }, "weight_granularity": { "_data": { "axis": 0 }, "_type": "PerAxis", "_version": 1 }, "weight_mapping_type": { "_data": "SYMMETRIC", "_type": "MappingType" }, "weight_scale_dtype": null }, "_type": "Int8DynamicActivationIntxWeightConfig", "_version": 2 } } }, "_type": "ModuleFqnToConfig", "_version": 1 } }, "quant_type_kwargs": {}, "untie_embedding_weights": false }, "text_config": { "_sliding_window_pattern": 6, "attention_bias": false, "attention_dropout": 0.0, "attn_logit_softcapping": null, "final_logit_softcapping": null, "head_dim": 256, "hidden_activation": "gelu_pytorch_tanh", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 10240, "layer_types": [ "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention" ], "max_position_embeddings": 131072, "model_type": "gemma3_text", "num_attention_heads": 8, "num_hidden_layers": 34, "num_key_value_heads": 4, "query_pre_attn_scalar": 256, "rms_norm_eps": 1e-06, "rope_local_base_freq": 10000.0, "rope_scaling": { "factor": 8.0, "rope_type": "linear" }, "rope_theta": 1000000.0, "sliding_window": 1024, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "use_cache": true, "vocab_size": 262208 }, "torch_dtype": "bfloat16", "transformers_version": "4.55.4", "vision_config": { "attention_dropout": 0.0, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 1152, "image_size": 896, "intermediate_size": 4304, "layer_norm_eps": 1e-06, "model_type": "siglip_vision_model", "num_attention_heads": 16, "num_channels": 3, "num_hidden_layers": 27, "patch_size": 14, "torch_dtype": "bfloat16", "vision_use_head": false } }