Remove base prefix from decimal numbers (closes #240)

Author: python-processing-unit

README.md

@@ -11,12 +11,12 @@ Prefix is a programming language focused on explicit, readable code.
 ! ---------- modular inverse via the extended Euclidean algorithm ----------
 func int modinv(int a, int m){
     a = MOD(a, m)
-    if(LT(a, 0d0)){ a = +(a, m) }
+    if(LT(a, 0)){ a = +(a, m) }
     int r0 = m
     int r1 = a
-    int t0 = 0d0
-    int t1 = 0d1
-    while(NEQ(r1, 0d0)){
+    int t0 = 0
+    int t1 = 1
+    while(NEQ(r1, 0)){
         int q = /(r0, r1)
         int nr = -(r0, *(q, r1))
         int nt = -(t0, *(q, t1))
@@ -25,11 +25,11 @@ func int modinv(int a, int m){
         t0 = t1
         t1 = nt
     }
-    if(NEQ(r0, 0d1)){
+    if(NEQ(r0, 0)){
         THROW("modinv: arguments not coprime")
     }
     int inv = MOD(t0, m)
-    if(LT(inv, 0d0)){ inv = +(inv, m) }
+    if(LT(inv, 0)){ inv = +(inv, m) }
     return(inv)
 }
 
@@ -38,9 +38,9 @@ func int modinv(int a, int m){
 ! The modulus is kept below 2^31 so that products of two consecutive
 ! differences (each < m) never overflow Prefix's signed 64-bit int.
 int m = 0x186A1
-int a = 0d16807
-int c = 0d12345
-int seed = 0d48271
+int a = 16807
+int c = 12345
+int seed = 48271
 
 func int step(int s){
     return(MOD(+( *(a, s), c), m))
@@ -49,7 +49,7 @@ func int step(int s){
 ! ---------- collect observed outputs (all the attacker sees) ----------
 map obs
 int s = seed
-for(i, 0d8){
+for(i, 8){
     obs<i> = s
     s = step(s)
 }
@@ -61,18 +61,18 @@ DEL("s")
 ! u_k = t_{k+1} * t_{k-1} - t_k^2 is a multiple of m.  GCD of several
 ! |u_k| yields m.
 map d
-for(i, 0d7){
-    d<i> = -(obs<+(i, 0d1)>, obs<i>)
+for(i, 7){
+    d<i> = -(obs<+(i, 1)>, obs<i>)
 }
 
-int mrec = 0d0
-int k = 0d2
-while(LTE(k, 0d6)){
-    int lo = -(k, 0d1)
-    int hi = +(k, 0d1)
+int mrec = 0
+int k = 2
+while(LTE(k, 6)){
+    int lo = -(k, 1)
+    int hi = +(k, 1)
     int tk = d<k>
     mrec = GCD(mrec, ABS(-( *(d<hi>, d<lo>), *(tk, tk) )))
-    k = +(k, 0d1)
+    k = +(k, 1)
 }
 DEL("d")
 DEL("k")
@@ -100,11 +100,11 @@ DEL("x1")
 DEL("modinv")
 
 ! ---------- verify by regenerating the whole sequence ----------
-int ok = 0d1
+int ok = 1
 int v = x0
-for(i, 0d8){
+for(i, 8){
     if(NEQ(v, obs<i>)){
-        ok = 0d0
+        ok = 0
     }
     v = MOD(+( *(arec, v), crec), mrec)
 }
@@ -124,7 +124,7 @@ DEL("step")
 PRINT("next output predicted: ", next_pred)
 PRINT("next output actual  : ", next_true)
 
-ASSERT(EQ(ok, 0d1))
+ASSERT(EQ(ok, 1))
 ASSERT(EQ(mrec, m))
 ASSERT(EQ(arec, a))
 ASSERT(EQ(crec, c))

docs/SPECIFICATION.html

@@ -64,9 +64,9 @@
 
 Identifiers MUST be non-empty and case-sensitive. Variables and user-defined functions share a single flat namespace, so one name MUST NOT denote both a variable and a function. A user-defined function name MUST NOT conflict with any built-in operator or function name.
 
-Identifiers MUST NOT contain non-ASCII characters or any of the following characters: `{`, `}`, `[`, `]`, `(`, `)`, `=`, `,`, `!`, `~`, or `@`. The first character of an identifier MUST NOT be `0`, and the first two characters of an identifier MUST NOT be `-0`.
+Identifiers MUST NOT contain non-ASCII characters or any of the following characters: `{`, `}`, `[`, `]`, `(`, `)`, `=`, `,`, `!`, `~`, or `@`. The first character of an identifier MUST NOT be a decimal digit (`0`-`9`), and the first two characters of an identifier MUST NOT be `-0`.
 
-The first identifier character MAY be a letter `A-Z` or `a-z`, a decimal digit `1-9`, `*`, or one of `/`, `$`, `%`, `&`, `_`, `+`, `|`, `?`, or `-`. If the first identifier character is `-`, the second character MAY be any otherwise valid identifier character except `0`. Subsequent identifier characters MAY additionally include the digit `0`, `*`, and `-`. The exact spelling `-INF` is reserved by [4.3.2](#432-special-values). This permissive ASCII-only character set preserves an unambiguous distinction between identifiers and numeric literals, which MUST begin with a `0`-prefixed base marker.
+The first identifier character MAY be a letter `A-Z` or `a-z`, `*`, or one of `/`, `$`, `%`, `&`, `_`, `+`, `|`, `?`, or `-`. If the first identifier character is `-`, the second character MUST be valid as an opening character. Subsequent identifier characters MAY additionally include the decimal digits `0-9`, `*`, and `-`. The exact spelling `-INF` is reserved by [4.3.2](#432-special-values). This permissive ASCII-only character set preserves an unambiguous distinction between identifiers and numeric literals, which MUST begin with a decimal digit.
 
 ---
 
@@ -268,7 +268,9 @@
 
 #### 4.2.1 Integer literals
 
-`int` literals MUST include a base prefix. Valid prefixes are `0b`, `0o`, `0d`, `0x`, `0t`, `0c`, `0s`, and `0rNN` where `NN` is two decimal digits and `2 <= NN <= 64`.
+Non-decimal `int` literals MUST include a base prefix. Valid prefixes are `0b`, `0o`, `0d`, `0x`, `0t`, `0c`, `0s`, and `0rNN` where `NN` is two decimal digits and `2 <= NN <= 64`.
+
+An `int` literal that begins with a decimal digit (`0`-`9`) and does NOT include a base prefix MUST be evaluated as a decimal (base-10) literal. A numeric literal that begins with a decimal digit and includes a base prefix (that is, one or more digits precede the base prefix) MUST raise an error.
 
 `int` literals MUST support negative values via prefixing with a `-` before the base prefix, and positive values with no sign. `int` literals MUST NOT support a `+` prefix.
 
@@ -290,9 +292,11 @@
 
 #### 4.3.1 Floating-point literals
 
-`float` literals MUST include a base prefix using the same prefix forms as `int` literals.
+Non-decimal `float` literals MUST include a base prefix using the same prefix forms as `int` literals.
+
+A `float` literal that begins with a decimal digit (`0`-`9`) and does NOT include a base prefix MUST be evaluated as a decimal (base-10) literal. A `float` literal that begins with a decimal digit and includes a base prefix (that is, one or more digits precede the base prefix) MUST raise an error.
 
-`float` literals MUST consist of a prefixed integer part, a radix point (`.`), and a fractional part. The integer and fractional parts MUST each contain at least one digit in the selected base.
+`float` literals MUST consist of an integer part (preceded by the base prefix, if present), a radix point (`.`), and a fractional part. The integer and fractional parts MUST each contain at least one digit in the selected base.
 
 `float` literals MUST support negative values via prefixing with a `-` before the base prefix, and positive values with no sign. `float` literals MUST NOT support a `+` prefix.
 

src/lexer.c

@@ -462,7 +462,7 @@ static bool is_number_body_char(char c) {
 }
 
 static bool is_identifier_start_char(char c) {
-    return strchr("abcdefghijklmnopqrstuvwxyz123456789/ABCDEFGHIJKLMNOPQRSTUVWXYZ$%&_+|?*", c) != NULL;
+    return strchr("abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ$%&_+|?*", c) != NULL;
 }
 
 static bool is_identifier_body_char(char c) {
@@ -650,7 +650,8 @@ Token lexer_next_token(Lexer *lexer) {
                 return number_token(lexer, true);
             }
 
-            if ((peek_next(lexer) == '0') || matches_reserved_signed_special(lexer, lookahead, "INF") ||
+            if ((isdigit((unsigned char)peek_next(lexer))) ||
+                matches_reserved_signed_special(lexer, lookahead, "INF") ||
                 matches_reserved_signed_special(lexer, lookahead, "NaN")) {
                 advance(lexer);
                 return make_token(lexer, TOKEN_DASH, "-", 1);
@@ -659,7 +660,7 @@ Token lexer_next_token(Lexer *lexer) {
             return identifier_token(lexer);
         }
 
-        if (c == '0' && is_base_prefix_char(peek_next(lexer))) {
+        if (isdigit((unsigned char)c)) {
             return number_token(lexer, false);
         }
 
@@ -752,40 +753,43 @@ static Token number_token(Lexer *lexer, bool is_negative_start) {
         value[len_val++] = '-';
     }
 
-    if (peek(lexer) != '0') {
-        free(value);
-        return error_token(lexer, "Invalid numeric literal; expected base prefix");
-    }
-    if (len_val + 1 >= capacity) {
-        capacity *= 2;
-        value = safe_realloc(value, capacity);
-    }
-    value[len_val++] = advance(lexer); // 0
+    bool prefixed = (peek(lexer) == '0' && is_base_prefix_char(peek_next(lexer)));
 
-    if (!is_base_prefix_char(peek(lexer))) {
+    if (is_negative_start && !prefixed) {
         free(value);
-        return error_token(lexer, "Invalid numeric literal base prefix");
-    }
-    if (len_val + 1 >= capacity) {
-        capacity *= 2;
-        value = safe_realloc(value, capacity);
+        return error_token(lexer, "Invalid numeric literal; expected base prefix");
     }
-    char pref = advance(lexer);
-    value[len_val++] = pref;
 
-    if (pref == 'r') {
-        if (!isdigit((unsigned char)peek(lexer)) || !isdigit((unsigned char)peek_next(lexer))) {
-            free(value);
-            return error_token(lexer, "0r literals require two decimal base digits");
+    if (prefixed) {
+        if (len_val + 1 >= capacity) {
+            capacity *= 2;
+            value = safe_realloc(value, capacity);
         }
-        if (len_val + 2 >= capacity) {
+        value[len_val++] = advance(lexer); // 0
+
+        if (len_val + 1 >= capacity) {
             capacity *= 2;
             value = safe_realloc(value, capacity);
         }
-        value[len_val++] = advance(lexer);
-        value[len_val++] = advance(lexer);
+        char pref = advance(lexer);
+        value[len_val++] = pref;
+
+        if (pref == 'r') {
+            if (!isdigit((unsigned char)peek(lexer)) || !isdigit((unsigned char)peek_next(lexer))) {
+                free(value);
+                return error_token(lexer, "0r literals require two decimal base digits");
+            }
+            if (len_val + 2 >= capacity) {
+                capacity *= 2;
+                value = safe_realloc(value, capacity);
+            }
+            value[len_val++] = advance(lexer);
+            value[len_val++] = advance(lexer);
+        }
     }
 
+    size_t digits_start = len_val;
+
     bool has_any = false;
     bool has_dot = false;
     bool has_frac = false;
@@ -841,6 +845,21 @@ static Token number_token(Lexer *lexer, bool is_negative_start) {
         return error_token(lexer, "Float literal requires fractional digits");
     }
 
+    if (!prefixed) {
+        for (size_t i = digits_start; i < len_val; i++) {
+            if (value[i] == '0' && i + 1 < len_val && is_base_prefix_char(value[i + 1])) {
+                free(value);
+                return error_token(lexer, "Numeric literal beginning with a digit must not include a base prefix");
+            }
+        }
+        for (size_t i = digits_start; i < len_val; i++) {
+            if (value[i] != '.' && !isdigit((unsigned char)value[i])) {
+                free(value);
+                return error_token(lexer, "Invalid digit in decimal literal");
+            }
+        }
+    }
+
     value[len_val] = '\0';
     Token t = {(int)has_dot ? TOKEN_FLOAT : TOKEN_NUMBER, value, start_line, start_col};
     return t;

src/parser.c

@@ -469,6 +469,10 @@ static int parse_prefixed_int_literal(const char *lit, int64_t *out_value, int *
 
     size_t prefix_len = 0;
     int base = base_from_literal_prefix(s, &prefix_len);
+    if (base < 0 && isdigit((unsigned char)s[0])) {
+        base = 10;
+        prefix_len = 0;
+    }
     if (base < 2 || base > 64) {
         return 0;
     }
@@ -510,6 +514,10 @@ static int parse_prefixed_float_literal(const char *lit, double *out_value, int
 
     size_t prefix_len = 0;
     int base = base_from_literal_prefix(s, &prefix_len);
+    if (base < 0 && isdigit((unsigned char)s[0])) {
+        base = 10;
+        prefix_len = 0;
+    }
     if (base < 2 || base > 64) {
         return 0;
     }

tests/cases/lexer/identifiers.yaml

@@ -54,15 +54,6 @@ passing:
         bool x = TRUE
         bool y = TRUE
         bool z = TRUE
-        bool 2 = TRUE
-        bool 3 = TRUE
-        bool 4 = TRUE
-        bool 5 = TRUE
-        bool 6 = TRUE
-        bool 7 = TRUE
-        bool 8 = TRUE
-        bool 9 = TRUE
-        bool 1 = TRUE
         bool / = TRUE
         bool $ = TRUE
         bool % = TRUE
@@ -106,3 +97,19 @@ failing:
     source: |
         bool 0x = TRUE
         
+  - name: digit-starts-ident
+    language: Prefix
+    source: |
+        bool 1x = TRUE
+        
+  - name: nine-starts-ident
+    language: Prefix
+    source: |
+        bool 9lives = TRUE
+        
+  - name: dash-before-digit
+    language: Prefix
+    source: |
+        bool -1abc = TRUE
+        
+

tests/cases/literals/flt.yaml

@@ -17,12 +17,33 @@ passing:
         ASSERT(EQ(d, 0xA.F))
         ASSERT(EQ(e, 0r16F.8))
         
+  - name: flt-decimal-no-prefix
+    language: Prefix
+    source: |
+        float a = 1.5
+        float b = 0.25
+        float c = 12.75
+        
+        ASSERT(EQ(a, 0d1.5))
+        ASSERT(EQ(b, 0d0.25))
+        ASSERT(EQ(c, 0d12.75))
+        
 failing:
   - name: flt-invalid-digit
     language: Prefix
     source: |
         float a = 0b2.0
         
+  - name: flt-digit-before-prefix
+    language: Prefix
+    source: |
+        float a = 10x1.F
+        
+  - name: flt-decimal-invalid-digit
+    language: Prefix
+    source: |
+        float a = 1.5F
+        
   - name: flt-negative-nan
     language: Prefix
     source: |