git ptrace.dev


        
            
            
        
        0#import "Basic";
1#import "Compiler";
2
3
4
5#module_parameters(CLEAN_TABLES_UP := true);
6
7
8
9serialize :: (input : *$T) -> []u8
10{
11    ALIGNED_INPUT_SIZE :: #run align_for_serialization(size_of(T));
12
13    dynamic_memory_required := cast(s64)handle_dynamic_memory(input, null, .COUNT); 
14
15    buffer : []u8;
16    buffer.count = ALIGNED_INPUT_SIZE + dynamic_memory_required;
17    buffer.data  = alloc(buffer.count);
18    assert(buffer.data == align_for_serialization(buffer.data));
19    memset(buffer.data, 0, buffer.count);
20
21    dynamic_memory : []u8;
22    dynamic_memory.count = dynamic_memory_required; 
23    dynamic_memory.data  = buffer.data + ALIGNED_INPUT_SIZE;
24
25    output := cast(*T)buffer.data;
26    memcpy(output, input, size_of(type_of(input.*)));
27
28    dynamic_memory_pointer := handle_dynamic_memory(output, dynamic_memory.data, .COPY_AND_ENCODE);
29    assert(dynamic_memory_pointer == dynamic_memory.data + dynamic_memory.count);
30
31    return buffer;
32}
33
34
35
36deserialize :: (bytes : []u8, $Output_Type : Type) -> *Output_Type  
37{
38    info := type_info(Output_Type);
39    result := cast(*Output_Type) bytes.data;
40    inline decode_dynamic_memory(result);
41    return result;
42}
43
44
45
46#scope_file
47
48
49
50#if CLEAN_TABLES_UP
51{
52    #import "Hash_Table";
53    cleanup_for_serialization :: (table : *Table)
54    {
55        DEFAULT_VALUE :: table.Entry.{};
56        for * table.entries
57            if it.hash < FIRST_VALID_HASH 
58                memcpy(it, *DEFAULT_VALUE, size_of(type_of(DEFAULT_VALUE)));
59    }
60}
61
62
63
64handle_dynamic_memory :: (element : *$T,
65                          dynamic_memory_pointer : *u8,
66                          $mode : enum { COUNT; COPY_AND_ENCODE; },
67                          $allow_pointer := false) -> next_dynamic_memory_pointer : *u8
68{
69    #insert #run () -> string
70    {
71        quick_string_builder();
72        p("// Handling dynamic memory for serializing %\n", T);
73        info := cast(*Type_Info)type_info(T);
74
75        #if mode == .COUNT
76        {
77            #if CLEAN_TABLES_UP if same_polymorph_source(info, type_info(Table(int,int)))
78            {
79                a(#string DONE
80                cleanup_for_serialization(element);
81                DONE);
82            }
83        }
84
85        if #complete info.type ==
86        {
87            case .STRING;
88            #if mode == .COPY_AND_ENCODE
89            {
90                a(#string DONE
91                memcpy(dynamic_memory_pointer, element.data, element.count);            // copy original data
92                element.data = xx (cast(u64)dynamic_memory_pointer - cast(u64)element); // make pointer, point to the dynamic buffer (encoded)
93                DONE);
94            }
95            a(#string DONE
96            dynamic_memory_pointer = align_for_serialization(dynamic_memory_pointer + element.count); // commit the dynamic allocation
97            DONE);
98
99
100
101            case .ARRAY;
102            array_info   := cast(*Type_Info_Array)info;
103            element_size := array_info.element_type.runtime_size;
104            if #complete array_info.array_type == 
105            {
106                case .RESIZABLE;
107                #if mode == .COPY_AND_ENCODE
108                {
109                    a(#string DONE
110                    element.allocated = element.count;
111                    element.allocator = Serialized_Data_Allocator;
112                    DONE);
113                }
114                #through;
115
116
117                case .VIEW;
118                a(#string DONE
119                array_data_size := element.count * size_of(type_of(element.data.*)); 
120                DONE);
121                #if mode == .COPY_AND_ENCODE
122                {
123                    a(#string DONE
124                    memcpy(dynamic_memory_pointer, element.data, array_data_size);           // copy original array elements
125                    element.data = xx (cast(u64)dynamic_memory_pointer - cast(u64)element);  // encode the data pointer 
126                    data_start := cast(type_of(element.data)) dynamic_memory_pointer;        // get a pointer to the data to iterate
127                    DONE);
128                }
129                else
130                {
131                    a(#string DONE
132                    data_start := element.data; // if we're not copying we just iterate throught the original data
133                    DONE);
134                }
135                a(#string DONE
136                dynamic_memory_pointer = align_for_serialization(dynamic_memory_pointer + array_data_size); // commit the dynamic allocation
137                for 0..element.count-1
138                {
139                    dynamic_memory_pointer = handle_dynamic_memory(data_start + it, dynamic_memory_pointer, mode); 
140                }
141                DONE);
142
143
144                case .FIXED;
145                a(#string DONE
146                for 0..element.count-1
147                {
148                    dynamic_memory_pointer = handle_dynamic_memory(element.data + it, dynamic_memory_pointer, mode); 
149                }
150                DONE);
151            }
152
153
154            
155            case .STRUCT;
156            struct_info := cast(*Type_Info_Struct)info;
157            
158            replace := needs_serialization_replacement(struct_info);
159            if replace
160            {
161                a(#string DONE
162                replaced_element := element.serialization_replacement(element);
163                #assert(size_of(T) == size_of(*element.Replacement_Type));
164                DONE);
165                #if mode == .COUNT
166                {
167                    a(#string DONE
168                    pointer_to_replaced_element := *replaced_element;
169                    dynamic_memory_pointer = handle_dynamic_memory(*pointer_to_replaced_element, dynamic_memory_pointer, mode, allow_pointer = true);
170                    DONE);
171                }
172                else
173                {
174                    a(#string DONE
175                    (cast(**element.Replacement_Type)element).* = *replaced_element;
176                    dynamic_memory_pointer = handle_dynamic_memory(cast(**element.Replacement_Type)element, dynamic_memory_pointer, mode, allow_pointer = true);
177                    DONE);
178                }
179            }
180            else
181            {
182                for member : struct_info.members
183                {
184                    if !(member.flags & .CONSTANT)
185                    {
186                        allow_pointer := false;
187                        for member.notes if it == "allow_single_element_pointer_serialization" then allow_pointer = true;
188                        p(#string DONE
189                        dynamic_memory_pointer = handle_dynamic_memory(cast(*type_of(T.%1))((cast(*u8)element) + %2), 
190                                                                       dynamic_memory_pointer,
191                                                                       mode,
192                                                                       allow_pointer = %3); 
193                        DONE, member.name, member.offset_in_bytes, allow_pointer);
194                    }
195                }
196            }
197
198
199
200            case .PROCEDURE;
201            if info != type_info(Allocator_Proc)
202            {
203                compiler_report(tprint("Only Allocator procedures can be in serialized structs, and % isn't one", get_type(info)));
204            }
205
206
207
208            case .POINTER;
209            a(#string DONE
210            if element.* != null
211            {
212            DONE);
213            #if allow_pointer
214            {
215                a(#string DONE
216                element_size := size_of(type_of((element.*).*));
217                DONE);
218                #if mode == .COPY_AND_ENCODE
219                {
220                    a(#string DONE
221                    memcpy(dynamic_memory_pointer, element.*, element_size);                      // copy original data
222                    (cast(*u64)element).* = (cast(u64)dynamic_memory_pointer - cast(u64)element); // make pointer, point to the dynamic buffer (encoded)
223                    element_data := cast(type_of(element.*)) dynamic_memory_pointer;
224                    DONE);
225                }
226                else
227                {
228                    a(#string DONE
229                    element_data := element.*;
230                    DONE);
231                }
232                a(#string DONE
233                dynamic_memory_pointer = align_for_serialization(dynamic_memory_pointer + element_size); // commit the dynamic allocation
234                dynamic_memory_pointer = handle_dynamic_memory(element_data, dynamic_memory_pointer, mode); 
235                DONE);
236            }
237            else
238            {
239                a(#string DONE
240                log_error("We only allow null-pointers to be serialized, and % wasn't!", T);
241                assert(false);
242                DONE);
243            }
244            a(#string DONE
245            } // if element.* != null
246            DONE);
247
248
249
250            case .VARIANT; 
251            a(#string DONE
252            INTERNAL_TYPE :: #run find_variant_contained_type(type_of(element.*));
253            dynamic_memory_pointer = handle_dynamic_memory(cast(*INTERNAL_TYPE)element, dynamic_memory_pointer, mode, allow_pointer);
254            DONE);
255
256
257
258            case .INTEGER;
259            case .FLOAT;
260            case .BOOL;
261            case .VOID;
262            case .POLYMORPHIC_VARIABLE;
263            case .ENUM;
264            // Nothing to do for these, they will just be memcpy-ed
265
266
267
268            case .TYPE;            #through;
269            case .CODE;            #through;
270            case .UNTYPED_ENUM;    #through;
271            case .UNTYPED_LITERAL; #through;
272            case .OVERLOAD_SET;    #through;
273            case .ANY;
274            compiler_report(tprint("Type % cannot be serialized cause it's a %", T, info.type));
275        }
276        return builder_to_string(b);
277    }();
278
279    return dynamic_memory_pointer;
280}
281
282
283
284decode_dynamic_memory :: (element : *$T)
285{
286    #insert #run () -> string
287    {
288        quick_string_builder();
289        p("// Decoding dynamic memory for deserializing %\n", T);
290        info := cast(*Type_Info)type_info(T);
291        if info.type ==
292        {
293            case .STRING;
294            a(#string DONE
295            element.data = cast(*u8)(cast(u64)element + cast(u64)element.data);
296            DONE);
297
298
299
300            case .ARRAY;
301            array_info   := cast(*Type_Info_Array)info;
302            element_type := array_info.element_type;
303            do_decode    := !(element_type.type == .INTEGER || element_type.type == .FLOAT || element_type.type == .BOOL);
304            if #complete array_info.array_type == 
305            {
306                case .RESIZABLE;
307                a(#string DONE
308                element.allocated = element.count;
309                element.allocator = Serialized_Data_Allocator;
310                DONE);
311                #through;
312
313
314                case .VIEW;
315                a(#string DONE
316                element.data = xx (cast(u64)element + cast(u64)element.data);
317                DONE);
318                #through;
319
320
321                case .FIXED;
322                if do_decode
323                {
324                    a(#string DONE
325                    for 0..element.count-1
326                    {
327                        inline decode_dynamic_memory(element.data + it); 
328                    }
329                    DONE);
330                }
331            }
332
333
334            
335            case .STRUCT;
336            struct_info := cast(*Type_Info_Struct)info;
337            replace := needs_serialization_replacement(struct_info);
338            if replace
339            {
340                a(#string DONE
341                #assert(size_of(T) == size_of(*element.Replacement_Type));
342                pointer_to_replaced_element : **element.Replacement_Type = cast(**element.Replacement_Type)element;
343                inline decode_dynamic_memory(pointer_to_replaced_element); 
344                element.* = element.deserialization_replacement(pointer_to_replaced_element.*); 
345                DONE);
346            }
347            for member : struct_info.members
348            {
349                if !(member.flags & .CONSTANT)
350                {
351                    p(#string DONE
352                    inline decode_dynamic_memory(cast(*type_of(T.%1))((cast(*u8)element) + %2));
353                    DONE, member.name, member.offset_in_bytes);
354                }
355            }
356
357
358
359            case .PROCEDURE;
360            if info == type_info(Allocator_Proc)
361            {
362                // @@NOTE: We could catch Allocator structs on the .STRUCT codepath
363                // instead, but just in case someone does an allocator procedure by itself
364                // maybe this is better and simpler.
365                p(#string DONE
366                element.* = Serialized_Data_Allocator.proc;
367                DONE);
368            }
369
370
371
372            case .POINTER;
373            a(#string DONE
374            offset := (cast(*u64)element).*;
375            if offset > 0
376            {
377                element.* = cast(T)(cast(u64)element + (cast(*u64)element).*);
378                inline decode_dynamic_memory(element.*); 
379            }
380            DONE);
381
382
383
384            case .VARIANT;
385            a(#string DONE
386            INTERNAL_TYPE :: #run find_variant_contained_type(type_of(element.*));
387            decode_dynamic_memory(cast(*INTERNAL_TYPE)element);
388            DONE);
389
390
391
392        }
393        return builder_to_string(b);
394    }();
395} 
396
397
398
399Serialized_Data_Allocator :: Allocator.{
400    proc = (mode: Allocator_Mode, size: s64, old_size: s64, old_memory_pointer: *void, allocator_data: *void) -> *void
401    {
402        log_error("Can't allocate data on a serialized data structure!");
403        assert(false);
404        return null;
405    },
406    data = null,
407};
408
409
410
411needs_serialization_replacement :: (struct_info : *Type_Info_Struct) -> bool
412{
413    found_type := false; 
414    found_serialization_replacement := false;
415    found_deserialization_replacement := false;
416    for member : struct_info.members
417    {
418        if member.flags & .CONSTANT
419        {
420            if member.name == "Replacement_Type" && member.type.type == .TYPE
421            {
422                found_type = true;
423            }
424            if member.name == "serialization_replacement" && member.type.type == .PROCEDURE
425            {
426                found_serialization_replacement = true;
427            }
428            if member.name == "deserialization_replacement" && member.type.type == .PROCEDURE
429            {
430                found_deserialization_replacement = true;
431            }
432        }
433    }
434    return found_type && found_serialization_replacement && found_deserialization_replacement;
435}
436
437
438
439//
440// @@NOTE: We'd like to handle the alignment of the serialized elements
441// explicitly, so if we're gonna copy an instance of some type into
442// dynamic memory, we'd first align the address via something like
443// an `alignment_of(T)`. However there's no way to do this currently.
444//
445// Also, when we allocate a buffer we also would have to align the allocation
446// and give the user back the unaligned addresse to free it, similar to how
447// `NewArray` works at the time of writing, which kind of sucks as an interface. 
448// Otherwise if we just alloc and don't align we'd be further aligning as we
449// encode the dynamic memory and leave things out of whack.
450//
451// So for now, since most defaults allocators align to 8 bytes, we're also gonna
452// align to 8 bytes here. All basic data types and pointers would just work. It'd
453// fail for SIMD registers that require larger alignments, but oh well, we don't have
454// real control of this here, unless we just aligned everything to 512 bits / 64 bytes 
455// to account for 512-wide SIMD ops (I believe the largest reasonable used one at the
456// time of writing). But that would also be a waste of space in the types that don't
457// require alignment.
458//
459// TLDR: We'd love an `alignment_of(T)` or an alignment field in `Type_Info` as well
460// as a way to `alloc(bytes, alignment)`.
461//
462//                                           - Ruben Osorio, 30/08/2025
463//
464align_for_serialization :: (value : $T) -> T
465{
466    return (value + 7) & (~7);
467}
468
469
470
471quick_string_builder :: () #expand
472{
473    `_builder : String_Builder;
474    _builder.allocator = temporary_allocator;
475    `defer reset(*_builder);
476    `b := *_builder;
477    `p :: (format_string : string, args: ..Any) #expand
478    {
479        print_to_builder(*_builder, format_string, ..args);
480    } @PrintLike
481    `a :: (the_string : string) #expand
482    {
483        append(*_builder, the_string);
484    }
485}
486
487
488
489find_variant_contained_type :: ($variant_type : Type) -> Type
490{
491    info := type_info(variant_type);
492    assert(info.type == .VARIANT);
493    variant_info := cast(*Type_Info_Variant)info;
494    return get_type(variant_info.variant_of);
495}
496
497
498
499same_polymorph_source :: (info_a : *Type_Info, info_b : *Type_Info) -> bool
500{
501    find_top_polymorph_source :: (info : *Type_Info) -> *Type_Info
502    {
503        while true
504        {
505            if info.type == .STRUCT
506            {
507                struct_info := cast(*Type_Info_Struct)info;
508                if struct_info.polymorph_source_struct
509                {
510                    info = struct_info.polymorph_source_struct;
511                    continue;
512                }
513            }
514            break;
515        }
516        return info;
517    }
518    return find_top_polymorph_source(info_a) == find_top_polymorph_source(info_b);
519}
520
521
522
index : blog
