Tokenizer ggcc

CMakeLists.txt

@@ -496,9 +496,14 @@ add_library(libfalcon
     libfalcon.h
     llama-util.h
     )
+add_library(cmpnct_unicode
+    cmpnct_unicode.cpp
+    cmpnct_unicode.h
+    )
 target_include_directories(libfalcon PUBLIC .)
 target_compile_features(libfalcon PUBLIC cxx_std_11) # don't bump
 target_link_libraries(libfalcon PRIVATE
+    cmpnct_unicode
     ggml
     ${LLAMA_EXTRA_LIBS}
     )

Makefile

@@ -251,7 +251,10 @@ ggml.o: ggml.c ggml.h ggml-cuda.h
 llama.o: llama.cpp ggml.h ggml-cuda.h llama.h llama-util.h
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
-libfalcon.o: libfalcon.cpp ggml.h ggml-cuda.h libfalcon.h llama-util.h
+cmpnct_unicode.o cmpnct_unicode.cpp cmpnct_unicode.h
+	$(CXX) $(CXXFLAGS) -c $< -o $@
+
+libfalcon.o: libfalcon.cpp ggml.h ggml-cuda.h libfalcon.h llama-util.h cmpnct_unicode.h
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
 common.o: examples/common.cpp examples/common.h
@@ -282,17 +285,17 @@ simple: examples/simple/simple.cpp                            build-info.h ggml.
 quantize: examples/quantize/quantize.cpp                      build-info.h ggml.o llama.o $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
 
-falcon_main: examples/falcon/falcon_main.cpp                                  build-info.h ggml.o libfalcon.o falcon_common.o $(OBJS)
+falcon_main: examples/falcon/falcon_main.cpp                                  build-info.h ggml.o libfalcon.o libfalcon.o falcon_common.o $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
 	@echo
 	@echo '====  Run ./falcon_main -h for help.  ===='
 	@echo 'Read the readme file for important parameters and usage'
 	@echo
 
-falcon_quantize: examples/falcon_quantize/quantize.cpp                      build-info.h ggml.o libfalcon.o $(OBJS)
+falcon_quantize: examples/falcon_quantize/quantize.cpp                      build-info.h ggml.o libfalcon.o libfalcon.o $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
 
-falcon_perplexity: examples/falcon_perplexity/falcon_perplexity.cpp                build-info.h ggml.o libfalcon.o falcon_common.o $(OBJS)
+falcon_perplexity: examples/falcon_perplexity/falcon_perplexity.cpp                build-info.h ggml.o libfalcon.o libfalcon.o falcon_common.o $(OBJS)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
 
 

cmpnct_unicode.h

@@ -0,0 +1,56 @@
+#ifndef CNCT_UNICODE_H
+#define CNCT_UNICODE_H
+
+#include <string>
+#include <vector>
+#include "cmpnct_unicode.h"
+enum CNCTCharType {
+    DIGIT,          // a numerical char in any language
+    LETTER,         // a letter in any language
+    WHITESPACE,     // any form of whitespace
+    ACCENT_MARK,    // letter modifiers like ´ in é
+    PUNCTUATION,    // punctuation including brackets
+    SYMBOL,         // math, currency, other symbols
+    CONTROL,        // control characters
+    MIXED,          // a mix of the above
+    UNIDENTIFIED    // something more exotic like emoji or separators
+};
+
+struct CNCTUnicode;
+
+struct CNCTString {
+    std::string str;
+    size_t utf8_chars;
+
+    CNCTCharType char_type=UNIDENTIFIED;
+    bool is_sequential=false;
+
+    size_t seq_offset_bytes=0;
+    size_t seq_offset_utf8_chars=0;
+
+    bool operator==(const std::string &other) const;
+    bool operator==(const char other) const;
+    bool operator==(const CNCTString &other) const;
+    CNCTString &operator+=(const std::string &other);
+    CNCTString &operator+=(const char other);
+    friend CNCTString operator+(CNCTString lhs, const std::string &rhs);
+    friend CNCTString operator+(CNCTString lhs, const char rhs);
+    CNCTString& operator+=(const CNCTString& other);
+    friend CNCTString operator+(CNCTString lhs, const CNCTString& rhs);
+};
+
+
+
+struct CNCTUnicode {
+    static bool check_code_range(int c, const std::vector<std::pair<int, int>>& ranges);
+    static CNCTCharType get_code_type(int c);
+    static CNCTCharType get_code_type(const std::string &utf8_char);
+    static int utf8_len(const char c);
+    static int strlen_utf8(std::string src);
+    static std::vector<std::string> split_utf8(const std::string &src);
+    static std::vector<CNCTString> split_utf8_enhanced(const std::string &src);
+    static CNCTCharType string_identify(const std::string& str);
+    static bool string_test(const std::string& str, CNCTCharType chartype);
+};
+
+#endif // CNCT_UNICODE_H

examples/falcon/falcon_main.cpp

@@ -139,7 +139,7 @@ int main(int argc, char ** argv) {
         params.prompt = gpt_random_prompt(rng);
     }
 
-    llama_init_backend();
+    falcon_init_backend();
 
     falcon_context * ctx;
     g_ctx = &ctx;
@@ -288,7 +288,8 @@ int main(int argc, char ** argv) {
     }
 
     // determine newline token
-    auto llama_token_newline = ::falcon_tokenize(ctx, "\n", false);
+    //auto llama_token_newline = ::falcon_tokenize(ctx, "\n", false);
+    auto llama_token_newline = std::vector<llama_token>(193);
 
     if (params.verbose_prompt) {
         fprintf(stderr, "\n");
@@ -711,8 +712,11 @@ fprintf(stderr, "+------------+-------+-------+-------+-------+---------------+\
             if (params.instruct) {
                 is_interacting = true;
             } else {
-                fprintf(stderr, " [end of text]\n");
-                break;
+                // fprintf(stderr, " [end of text]\n");
+                // if we are in the prompt ingestion we will not stop
+                if (n_past > embd_inp.size()) {
+                    break;
+                }
             }
         }
 

examples/falcon_common.cpp

@@ -127,7 +127,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
 
         if (arg == "-s" || arg == "--seed") {
 #if defined(GGML_USE_CUBLAS)
-            fprintf(stderr, "WARNING: when using cuBLAS generation results are NOT guaranteed to be reproducible.\n");
+            // fprintf(stderr, "WARNING: when using cuBLAS generation results are NOT guaranteed to be reproducible.\n");
 #endif
             if (++i >= argc) {
                 invalid_param = true;

examples/falcon_perplexity/falcon_perplexity.cpp

@@ -147,7 +147,7 @@ int main(int argc, char ** argv) {
         params.prompt = gpt_random_prompt(rng);
     }
 
-    llama_init_backend();
+    falcon_init_backend();
 
     falcon_context * ctx;
 

examples/falcon_quantize/quantize.cpp

@@ -7,6 +7,7 @@
 #include <vector>
 #include <string>
 
+#define FILE_VERSION_S "GGCC Falcon V1"
 struct quant_option {
     std::string name;
     llama_ftype ftype;
@@ -17,99 +18,99 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {
     {
         "Q4_0",
         LLAMA_FTYPE_MOSTLY_Q4_0,
-        " 3.50G, +0.2499 ppl @ 7B - small, very high quality loss - legacy, prefer using Q3_K_M",
+        " - small, very high quality loss",
     },
     {
         "Q4_1",
         LLAMA_FTYPE_MOSTLY_Q4_1,
-        " 3.90G, +0.1846 ppl @ 7B - small, substantial quality loss - legacy, prefer using Q3_K_L",
+        " - small, substantial quality loss",
     },
     {
         "Q5_0",
         LLAMA_FTYPE_MOSTLY_Q5_0,
-        " 4.30G, +0.0796 ppl @ 7B - medium, balanced quality - legacy, prefer using Q4_K_M",
+        " - medium, balanced quality",
     },
     {
         "Q5_1",
         LLAMA_FTYPE_MOSTLY_Q5_1,
-        " 4.70G, +0.0415 ppl @ 7B - medium, low quality loss - legacy, prefer using Q5_K_M",
+        " - medium, low quality loss",
     },
 #ifdef GGML_USE_K_QUANTS
     {
         "Q2_K",
         LLAMA_FTYPE_MOSTLY_Q2_K,
-        " 2.67G, +0.8698 ppl @ 7B - smallest, extreme quality loss - not recommended",
+        " - smallest, very high quality loss - try for yourself",
     },
     {
         "Q3_K",
         LLAMA_FTYPE_MOSTLY_Q3_K_M,
-        "alias for Q3_K_M"
-    },
-    {
-        "Q3_K_S",
-        LLAMA_FTYPE_MOSTLY_Q3_K_S,
-        " 2.75G, +0.5505 ppl @ 7B - very small, very high quality loss",
-    },
-    {
-        "Q3_K_M",
-        LLAMA_FTYPE_MOSTLY_Q3_K_M,
-        " 3.06G, +0.2437 ppl @ 7B - very small, very high quality loss",
-    },
-    {
-        "Q3_K_L",
-        LLAMA_FTYPE_MOSTLY_Q3_K_L,
-        " 3.35G, +0.1803 ppl @ 7B - small, substantial quality loss",
+        " - very small, very high quality loss"
     },
+    // {
+    //     "Q3_K_S",
+    //     LLAMA_FTYPE_MOSTLY_Q3_K_S,
+    //     " 2.75G, +0.5505 ppl @ 7B - very small, very high quality loss",
+    // },
+    // {
+    //     "Q3_K_M",
+    //     LLAMA_FTYPE_MOSTLY_Q3_K_M,
+    //     " 3.06G, +0.2437 ppl @ 7B - very small, very high quality loss",
+    // },
+    // {
+    //     "Q3_K_L",
+    //     LLAMA_FTYPE_MOSTLY_Q3_K_L,
+    //     " 3.35G, +0.1803 ppl @ 7B - small, substantial quality loss",
+    // },
     {
         "Q4_K",
         LLAMA_FTYPE_MOSTLY_Q4_K_M,
-        "alias for Q4_K_M",
-    },
-    {
-        "Q4_K_S",
-        LLAMA_FTYPE_MOSTLY_Q4_K_S,
-        " 3.56G, +0.1149 ppl @ 7B - small, significant quality loss",
-    },
-    {
-        "Q4_K_M",
-        LLAMA_FTYPE_MOSTLY_Q4_K_M,
-        " 3.80G, +0.0535 ppl @ 7B - medium, balanced quality - *recommended*",
+        " - medium, balanced quality - *recommended*",
     },
+    // {
+    //     "Q4_K_S",
+    //     LLAMA_FTYPE_MOSTLY_Q4_K_S,
+    //     " 3.56G, +0.1149 ppl @ 7B - small, significant quality loss",
+    // },
+    // {
+    //     "Q4_K_M",
+    //     LLAMA_FTYPE_MOSTLY_Q4_K_M,
+    //     " 3.80G, +0.0535 ppl @ 7B - medium, balanced quality - *recommended*",
+    // },
     {
         "Q5_K",
         LLAMA_FTYPE_MOSTLY_Q5_K_M,
-        "alias for Q5_K_M",
-    },
-    {
-        "Q5_K_S",
-        LLAMA_FTYPE_MOSTLY_Q5_K_S,
-        " 4.33G, +0.0353 ppl @ 7B - large, low quality loss - *recommended*",
-    },
-    {
-        "Q5_K_M",
-        LLAMA_FTYPE_MOSTLY_Q5_K_M,
-        " 4.45G, +0.0142 ppl @ 7B - large, very low quality loss - *recommended*",
+        " - large, very low quality loss - *recommended*",
     },
+    // {
+    //     "Q5_K_S",
+    //     LLAMA_FTYPE_MOSTLY_Q5_K_S,
+    //     " 4.33G, +0.0353 ppl @ 7B - large, low quality loss - *recommended*",
+    // },
+    // {
+    //     "Q5_K_M",
+    //     LLAMA_FTYPE_MOSTLY_Q5_K_M,
+    //     " 4.45G, +0.0142 ppl @ 7B - large, very low quality loss - *recommended*",
+    // },
     {
         "Q6_K",
         LLAMA_FTYPE_MOSTLY_Q6_K,
-        " 5.15G, +0.0044 ppl @ 7B - very large, extremely low quality loss",
+        " - very large, extremely low quality loss",
     },
 #endif
     {
         "Q8_0",
         LLAMA_FTYPE_MOSTLY_Q8_0,
-        " 6.70G, +0.0004 ppl @ 7B - very large, extremely low quality loss - not recommended",
+        " - very large, extremely low quality loss - not recommended",
     },
     {
         "F16",
         LLAMA_FTYPE_MOSTLY_F16,
-        "13.00G              @ 7B - extremely large, virtually no quality loss - not recommended",
+        " - extremely large, virtually no quality loss - for quality benchmark",
     },
     {
         "F32",
         LLAMA_FTYPE_ALL_F32,
-        "26.00G              @ 7B - absolutely huge, lossless - not recommended",
+        " - absolutely huge, lossless - not recommended",
     },
 };
 
@@ -147,6 +148,7 @@ bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftype, std:
 //  ./quantize [--allow-requantize] [--leave-output-tensor] models/llama/ggml-model.bin [models/llama/ggml-model-quant.bin] type [nthreads]
 //
 void usage(const char * executable) {
+    fprintf(stderr, "Falcon - ggllm.cpp quantizer. Version %s \n", FILE_VERSION_S);
     fprintf(stderr, "usage: %s [--help] [--allow-requantize] [--leave-output-tensor] model-f32.bin [model-quant.bin] type [nthreads]\n\n", executable);
     fprintf(stderr, "  --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n");
     fprintf(stderr, "  --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n");
@@ -180,7 +182,7 @@ int main(int argc, char ** argv) {
         usage(argv[0]);
     }
 
-    llama_init_backend();
+    falcon_init_backend();
 
     // parse command line arguments
     const std::string fname_inp = argv[arg_idx];

ggml-cuda.cu

@@ -2080,6 +2080,9 @@ void * ggml_cuda_host_malloc(size_t size) {
     if (getenv("GGML_CUDA_NO_PINNED") != nullptr) {
         return nullptr;
     }
+    if (g_system_gpu_status.num_devices == 0) {
+        return nullptr;
+    }
 
     void * ptr = nullptr;
     cudaError_t err = cudaMallocHost((void **) &ptr, size);

ggml.c

@@ -4136,6 +4136,7 @@ thread_ret_t ggml_init_cublas_wrapper(void *data)
     return 0;
 }
 #endif
+pthread_t initThread;
 struct ggml_context * ggml_init(struct ggml_init_params params) {
     // make this function thread safe
     ggml_critical_section_start();
@@ -4189,7 +4190,7 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {
             // get num devices and memory information to avoid a potential race condition
             ggml_cuda_update_gpu_status(-1);
             // launch the handle creation into the background)
-            pthread_t initThread;
+
             pthread_create(&initThread, NULL, ggml_init_cublas_wrapper, NULL);
             // init needs to be checked before first operation starts
         }