Refactor dotenv module so that it can be tested on host

jepler · jepler · commit 52bca952080e · 2022-10-04T09:51:27.000-05:00
.. it needs to operate on a FILE* rather than FIL depending on
the build.

Note that this is comparing output to expected, not to cpython dotenv
package. Because run-tests.py starts the CPython interpreter with the
'-S' (skip site initialization) flag, pip-installed packages are
not available for import inside a test file. Instead, the exp
file is generated manually:
```
circuitpython/tests$ python3 circuitpython/dotenv_test.py &gt; circuitpython/dotenv_test.py.exp
```

Unfortunately, the test fails on test e15:
```diff
FAILURE /home/jepler/src/circuitpython/tests/results/circuitpython_dotenv_test.py
--- /home/jepler/src/circuitpython/tests/results/circuitpython_dotenv_test.py.exp	2022-10-04 09:48:16.307703128 -0500
+++ /home/jepler/src/circuitpython/tests/results/circuitpython_dotenv_test.py.out	2022-10-04 09:48:16.307703128 -0500
@@ -14,7 +14,7 @@
 line
 e13 e13value
 e14 None
-e15 e15value
+e15 None
 e16 #
 e17 def
 e18 #has a hash
```
diff --git a/ports/unix/variants/coverage/mpconfigvariant.mk b/ports/unix/variants/coverage/mpconfigvariant.mk
@@ -33,6 +33,7 @@ SRC_BITMAP := \
 	shared-bindings/aesio/__init__.c \
 	shared-bindings/bitmaptools/__init__.c \
 	shared-bindings/displayio/Bitmap.c \
+	shared-bindings/dotenv/__init__.c \
 	shared-bindings/rainbowio/__init__.c \
 	shared-bindings/traceback/__init__.c \
 	shared-bindings/util.c \
@@ -44,17 +45,18 @@ SRC_BITMAP := \
 	shared-module/displayio/Bitmap.c \
 	shared-module/displayio/ColorConverter.c \
 	shared-module/displayio/ColorConverter.c \
+	shared-module/dotenv/__init__.c \
 	shared-module/rainbowio/__init__.c \
 	shared-module/traceback/__init__.c \
 	shared-module/zlib/__init__.c \
 
-$(info $(SRC_BITMAP))
 SRC_C += $(SRC_BITMAP)
 
 CFLAGS += \
 	-DCIRCUITPY_AESIO=1 \
 	-DCIRCUITPY_BITMAPTOOLS=1 \
 	-DCIRCUITPY_DISPLAYIO_UNIX=1 \
+	-DCIRCUITPY_DOTENV=1 \
 	-DCIRCUITPY_GIFIO=1 \
 	-DCIRCUITPY_RAINBOWIO=1 \
 	-DCIRCUITPY_TRACEBACK=1 \
diff --git a/shared-module/dotenv/__init__.c b/shared-module/dotenv/__init__.c
@@ -29,12 +29,45 @@
 
 #include "shared-bindings/dotenv/__init__.h"
 
-#include "extmod/vfs.h"
-#include "extmod/vfs_fat.h"
 #include "py/mpstate.h"
 #include "py/objstr.h"
 #include "supervisor/filesystem.h"
 
+#if defined(UNIX)
+typedef FILE *file_arg;
+STATIC bool open_file(const char *name, file_arg *active_file) {
+    FILE *result = fopen(name, "r");
+    if (result) {
+        *active_file = result;
+    }
+    return result != NULL;
+}
+STATIC void close_file(file_arg *active_file) {
+    fclose(*active_file);
+}
+STATIC uint8_t get_next_character(file_arg *active_file) {
+    int value = fgetc(*active_file);
+    if (value == EOF) {
+        return 0;
+    }
+    return value;
+}
+STATIC void seek_minus_one(file_arg *active_file) {
+    fseek(*active_file, -1, SEEK_CUR);
+}
+#else
+#include "extmod/vfs.h"
+#include "extmod/vfs_fat.h"
+typedef FIL file_arg;
+STATIC bool open_file(const char *name, file_arg *active_file) {
+    FATFS *fs = filesystem_circuitpy();
+    FRESULT result = f_open(fs, active_file, name, FA_READ);
+    return result == FR_OK;
+}
+STATIC void close_file(file_arg *active_file) {
+    // nothing
+}
+
 // Return 0 if there is no next character (EOF).
 STATIC uint8_t get_next_character(FIL *active_file) {
     uint8_t character = 0;
@@ -43,10 +76,14 @@ STATIC uint8_t get_next_character(FIL *active_file) {
     f_read(active_file, &character, 1, &quantity_read);
     return character;
 }
+STATIC void seek_minus_one(file_arg *active_file) {
+    f_lseek(active_file, f_tell(active_file) - 1);
+}
+#endif
 
 // Discard whitespace, except for newlines, returning the next character after the whitespace.
 // Return 0 if there is no next character (EOF).
-STATIC uint8_t consume_whitespace(FIL *active_file) {
+STATIC uint8_t consume_whitespace(file_arg *active_file) {
     uint8_t character;
     do {
         character = get_next_character(active_file);
@@ -56,7 +93,7 @@ STATIC uint8_t consume_whitespace(FIL *active_file) {
 
 // Starting at the start of a new line, determines if the key matches the given
 // key. File pointer is set to be just before the = after the key.
-STATIC bool key_matches(FIL *active_file, const char *key) {
+STATIC bool key_matches(file_arg *active_file, const char *key) {
     uint8_t character;
     character = consume_whitespace(active_file);
     if (character == 0) {
@@ -99,7 +136,7 @@ STATIC bool key_matches(FIL *active_file, const char *key) {
     }
     if (character == '=' || character == '\n' || character == '#' || character == 0) {
         // Rewind one so the value, if any, can be found.
-        f_lseek(active_file, f_tell(active_file) - 1);
+        seek_minus_one(active_file);
     } else {
         // We're followed by something else that is invalid syntax.
         matches = false;
@@ -108,7 +145,7 @@ STATIC bool key_matches(FIL *active_file, const char *key) {
     return matches && key_pos == key_len;
 }
 
-STATIC bool next_line(FIL *active_file) {
+STATIC bool next_line(file_arg *active_file) {
     uint8_t character;
     bool quoted = false;
     bool escaped = false;
@@ -135,7 +172,7 @@ STATIC bool next_line(FIL *active_file) {
     return character != 0;
 }
 
-STATIC mp_int_t read_value(FIL *active_file, char *value, size_t value_len) {
+STATIC mp_int_t read_value(file_arg *active_file, char *value, size_t value_len) {
     uint8_t character;
     // Consume spaces before "=", and get first character of interest.
     character = consume_whitespace(active_file);
@@ -184,7 +221,7 @@ STATIC mp_int_t read_value(FIL *active_file, char *value, size_t value_len) {
         if (!quoted && (character == '\n' || (character == '#' && !first_char))) {
             if (character == '\n') {
                 // Rewind one so the next_line can find the \n.
-                f_lseek(active_file, f_tell(active_file) - 1);
+                seek_minus_one(active_file);
             }
             break;
         }
@@ -208,10 +245,8 @@ STATIC mp_int_t read_value(FIL *active_file, char *value, size_t value_len) {
 }
 
 mp_int_t dotenv_get_key(const char *path, const char *key, char *value, mp_int_t value_len) {
-    FIL active_file;
-    FATFS *fs = filesystem_circuitpy();
-    FRESULT result = f_open(fs, &active_file, path, FA_READ);
-    if (result != FR_OK) {
+    file_arg active_file;
+    if (!open_file(path, &active_file)) {
         return -1;
     }
 
@@ -224,7 +259,7 @@ mp_int_t dotenv_get_key(const char *path, const char *key, char *value, mp_int_t
 
         read_ok = next_line(&active_file);
     }
-    f_close(&active_file);
+    close_file(&active_file);
     return actual_value_len;
 }
 
diff --git a/tests/circuitpython/dotenv_test.env b/tests/circuitpython/dotenv_test.env
diff --git a/tests/circuitpython/dotenv_test.py b/tests/circuitpython/dotenv_test.py
@@ -1,6 +1,6 @@
 import dotenv
 
-FILE = "dotenv_test.env"
+FILE = __file__.rsplit(".", 1)[0] + ".env"
 
 print("e0", dotenv.get_key(FILE, "e0"))
 print("e1", dotenv.get_key(FILE, "e1"))
diff --git a/tests/circuitpython/dotenv_test.py.exp b/tests/circuitpython/dotenv_test.py.exp
@@ -0,0 +1,20 @@
+e0 None
+e1 e1value
+e2 e2value
+e3 e3value
+e4 e4value
+e5 None
+e6 
+e7 #
+e8 
+e9 #
+e10 e10_last
+e11 abc#def
+e12 multi
+line
+e13 e13value
+e14 None
+e15 e15value
+e16 #
+e17 def
+e18 #has a hash
diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp
@@ -32,18 +32,19 @@ mport
 builtins        micropython     _asyncio        _thread
 _uasyncio       aesio           array           binascii
 bitmaptools     btree           cexample        cmath
-collections     cppexample      displayio       errno
-ffi             framebuf        gc              gifio
-hashlib         json            math            qrio
-rainbowio       re              sys             termios
-traceback       ubinascii       uctypes         uerrno
-uheapq          uio             ujson           ulab
-ulab.numpy      ulab.numpy.fft  ulab.numpy.linalg
-ulab.scipy      ulab.scipy.linalg
-ulab.scipy.optimize             ulab.scipy.signal
-ulab.scipy.special              ulab.utils      uos
-urandom         ure             uselect         ustruct
-utime           utimeq          uzlib           zlib
+collections     cppexample      displayio       dotenv
+errno           ffi             framebuf        gc
+gifio           hashlib         json            math
+qrio            rainbowio       re              sys
+termios         traceback       ubinascii       uctypes
+uerrno          uheapq          uio             ujson
+ulab            ulab.numpy      ulab.numpy.fft
+ulab.numpy.linalg               ulab.scipy
+ulab.scipy.linalg               ulab.scipy.optimize
+ulab.scipy.signal               ulab.scipy.special
+ulab.utils      uos             urandom         ure
+uselect         ustruct         utime           utimeq
+uzlib           zlib
 ime
 
 utime           utimeq
