zfec

.gitignore

@@ -112,3 +112,5 @@ fvid/*.m4a
 fvid/*.txt
 fvid/*.jpg
 fvid/*.png
+fvid/*.lprof
+

docs/source/index.rst

@@ -10,8 +10,8 @@ Welcome to Fvid's documentation!
    :maxdepth: 2
    :caption: Contents:
 
-   history
    howto
+   history
 
 Download & Install
 ==================

fvid/fvid.py

@@ -11,6 +11,9 @@ import io
 import gzip
 import json
 import base64
+import decimal
+
+from zfec import easyfec as ef
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
@@ -34,6 +37,11 @@ NOTDEBUG = True
 TEMPVIDEO = "_temp.mp4"
 FRAMERATE = "1"
 
+# DO NOT CHANGE
+KVAL = 4
+MVAL = 5
+BLOCK = 16
+
 
 class WrongPassword(Exception):
     pass
@@ -67,21 +75,51 @@ def get_password(password_provided: str) -> bytes:
         key = kdf.derive(password)
         return key
 
-
-def get_bits_from_file(filepath: str, key: bytes) -> BitArray:
+def encode_zfec(bit_array: BitArray) -> BitArray:
+    global KVAL, MVAL, BLOCK
     """
-    Get/read bits from file
+    Apply Reed-Solomon error correction every byte to maximize retrieval
+    possibility as opposed to applying it to the entire file
+
+    bit_array -- BitArray containing raw file data
+    ecc -- The error correction value to be used (default DEFAULT_ECC)
+    """
+
+    bits = bit_array.bin
+
+    # split bits into blocks of bits
+    byte_list = split_string_by_n(bits, BLOCK)
+
+    ecc_bytes = ""
+
+
+    print("Applying Zfec Error Correction...")
+
+    encoder = ef.Encoder(KVAL, MVAL)
+    for b in tqdm(byte_list):
+        ecc_bytes += ''.join(map(bytes.decode, encoder.encode(b.encode('utf-8'))))
+
+    return BitArray(bytes=ecc_bytes.encode('utf-8'))
+
+def get_bits_from_file(
+    filepath: str, key: bytes, zfec: bool
+) -> BitArray:
+    """
+    Get/read bits fom file, encrypt data, and zip
 
     filepath -- the file to read
     key -- key used to encrypt file
+    zfec -- if reed solomon should be used to encode bits
     """
 
     print("Reading file...")
 
     bitarray = BitArray(filename=filepath)
 
-    # adding a delimiter to know when the file ends to avoid corrupted files
-    # when retrieving
+    if zfec:
+        bitarray = encode_zfec(bitarray)
+
+    # encrypt data
     cipher = AES.new(key, AES.MODE_EAX, nonce=SALT)
     ciphertext, tag = cipher.encrypt_and_digest(bitarray.tobytes())
 
@@ -89,6 +127,7 @@ def get_bits_from_file(filepath: str, key: bytes) -> BitArray:
 
     # because json can only serialize strings, the byte objects are encoded
     # using base64
+
     data_bytes = json.dumps(
         {
             "tag": base64.b64encode(tag).decode("utf-8"),
@@ -97,7 +136,7 @@ def get_bits_from_file(filepath: str, key: bytes) -> BitArray:
         }
     ).encode("utf-8")
 
-    print("Zipping...")
+    # print("Zipping...")
 
     # zip
     out = io.BytesIO()
@@ -109,6 +148,7 @@ def get_bits_from_file(filepath: str, key: bytes) -> BitArray:
     del bitarray
 
     bitarray = BitArray(zip)
+    # bitarray = BitArray(data_bytes)
 
     return bitarray.bin
 
@@ -185,19 +225,62 @@ def get_bits_from_video(video_filepath: str) -> str:
 
     if use_cython:
         print("Using Cython...")
+
     for index in tqdm(range(sequence_length)):
         bits += get_bits_from_image(image_sequence[index])
 
     return bits
 
 
-def save_bits_to_file(file_path: str, bits: str, key: bytes):
+def decode_zfec(data_bytes: bytes) -> bytes:
+    global KVAL, MVAL, BLOCK
+
+    KVAL = 4
+    MVAL = 5
+
+    byte_list = split_string_by_n(data_bytes, int(BLOCK*(MVAL/KVAL)))
+
+    # appending to a single bytes object is very slow so we make 50-51 and combine at the end
+    decoded_bytes = [bytes()] * (int(len(byte_list) / 50) + 1)
+
+    print("Decoding Zfec Error Correction...")
+
+    decoder = ef.Decoder(KVAL, MVAL)
+    i = 0
+    for b in tqdm(byte_list):
+        base = split_string_by_n(b, len(b) // MVAL)
+        idk = decoder.decode(base[:KVAL], list(range(KVAL)), 0)
+        idk2 = decoder.decode(base[1:KVAL+1], list(range(KVAL+1))[1:], 0)
+        if idk == idk2:
+            decoded_bytes[i//50] += idk
+        else: # its corrupted here
+            j = 10
+            while j > 0 and idk != idk2:
+                random.shuffle(base)
+                idk = decoder.decode(base[:KVAL], list(range(KVAL)), 0)
+                idk2 = decoder.decode(base[1:KVAL+1], list(range(KVAL+1))[1:], 0)
+                j -= 1
+            decoded_bytes[i//50] += idk # it should be correct by now
+        i += 1
+
+    decoded_bytes2 = bytes()
+    
+    for bytestring in tqdm(decoded_bytes):
+        decoded_bytes2 += bytestring
+
+    return decoded_bytes2
+
+
+def save_bits_to_file(
+    file_path: str, bits: str, key: bytes, zfec: bool
+):
     """
     save/write bits to a file
 
     file_path -- the path to write to
     bits -- the bits to write
     key -- key userd for file decryption
+    zfec -- needed if reed solomon was used to encode bits
     """
 
     bitstring = Bits(bin=bits)
@@ -220,8 +303,9 @@ def save_bits_to_file(file_path: str, bits: str, key: bytes):
 
     filename = data["filename"]
 
+    # decrypting data
     cipher = AES.new(key, AES.MODE_EAX, nonce=SALT)
-    bitstring = cipher.decrypt(ciphertext)
+    data_bytes = cipher.decrypt(ciphertext)
 
     print("Checking integrity...")
 
@@ -230,7 +314,12 @@ def save_bits_to_file(file_path: str, bits: str, key: bytes):
     except ValueError:
         raise WrongPassword("Key incorrect or message corrupted")
 
-    bitstring = BitArray(bitstring)
+    bitstring = Bits(data_bytes)
+
+    if zfec:
+        bitstring = Bits(
+            "0b" + decode_zfec(data_bytes).decode("utf-8")
+        )
 
     # If filepath not passed in use default otherwise used passed in filepath
     if file_path == None:
@@ -286,11 +375,12 @@ def make_image_sequence(bitstring: BitArray, resolution: tuple = (1920, 1080)):
 
     index = 1
     bitlist = bitlist[::-1]
+
     print("Saving frames...")
+
     for _ in tqdm(range(len(bitlist))):
         bitl = bitlist.pop()
         image_bits = list(map(int, bitl))
-        # print(image_bits)
 
         image = Image.new("1", (width, height))
         image.putdata(image_bits)
@@ -336,7 +426,6 @@ def setup():
     if not os.path.exists(FRAMES_DIR):
         os.makedirs(FRAMES_DIR)
 
-
 def main():
     global FRAMERATE
     parser = argparse.ArgumentParser(description="save files as videos")
@@ -364,6 +453,21 @@ def main():
         type=str,
         default="default",
     )
+    parser.add_argument(
+        "-z",
+        "--zfec",
+        help=(
+            "Apply Zfec error correcting. This is helpful if you're"
+            " finding that your data is not being decoded correctly. It adds"
+            " 2 extra bits per byte making it possible to recover all 8 bits"
+            " in the case the data changes during the decoding process at"
+            " the cost of making your video files larger. Note, if you use"
+            " this option, you must also use the -r flag to decode a video"
+            " back to a file, otherwise, your data will not be recovered"
+            " correctly."
+        ),
+        action="store_true",
+    )
 
     args = parser.parse_args()
 
@@ -398,7 +502,7 @@ def main():
         if args.output:
             file_path = args.output
 
-        save_bits_to_file(file_path, bits, key)
+        save_bits_to_file(file_path, bits, key, args.zfec)
 
     elif args.encode:
 
@@ -415,7 +519,7 @@ def main():
             )
 
         # get bits from file
-        bits = get_bits_from_file(args.input, key)
+        bits = get_bits_from_file(args.input, key, args.zfec)
 
         # create image sequence
         make_image_sequence(bits)
@@ -427,4 +531,7 @@ def main():
 
         make_video(video_file_path, args.framerate)
 
-    cleanup()
+    cleanup()
+
+if __name__ == '__main__':
+    main()