• 🏆 Texturing Contest #33 is OPEN! Contestants must re-texture a SD unit model found in-game (Warcraft 3 Classic), recreating the unit into a peaceful NPC version. 🔗Click here to enter!

[Snippet] HashAlloc/HashStruct - no instance limit

Status
Not open for further replies.

~El

Level 17
Joined
Jun 13, 2016
Messages
558
Hey everyone. Recently I was working on my map and I ended up needing to extend the max size of my struct beyond 8191 instances. Now, I didn't want to rewrite a lot of my code that used the struct's nice access syntax and switch it over to manual HashTable usage, so I wrote a small snippet to get struct-like syntax with hashtables underneath.

I decided to share it here, because I think it might be useful to some people in certain corner cases where you need a lot of data and a struct-like syntax. This way, you don't have to worry about going over the 8191 instance limit.

Allocation based on the ubiquotous Alloc library used almost by everyone. Uses Bribe's Table library for TableArrays. It uses TextMacros to set it up, which is a bit inconvenient, but setting up a HashAlloc struct doesn't look that terrible.

JASS:
library HashAlloc requires Table
    //! textmacro SetupHashAllocStart takes COUNT
        private static Table recycler
        private static TableArray fields

        static method allocate takes nothing returns thistype
            local thistype this = recycler[0]

            if (not recycler.has(this)) then
                set recycler[0] = this + 1
            else
                set recycler[0] = recycler[this]
            endif

            return this
        endmethod

        method deallocate takes nothing returns nothing
            set recycler[this] = recycler[0]
            set recycler[0] = this
        endmethod

        private static method onInit takes nothing returns nothing
            set recycler = Table.create()
            set recycler[0] = 1
            set fields = TableArray[$COUNT$]
    //! endtextmacro

    //! textmacro SetupHashAllocEnd
        endmethod
    //! endtextmacro

    //! textmacro SetupFieldThistype takes ID, NAME
        method operator $NAME$ takes nothing returns thistype
            return fields[$ID$].integer[this]
        endmethod

        method operator $NAME$= takes thistype arg returns nothing {
            set fields[$ID$].integer[this] = arg
        endmethod
    //! endtextmacro


    //! textmacro SetupField takes ID, NAME, TYPE
        method operator $NAME$ takes nothing returns $TYPE$ {
            return fields[$ID$].$TYPE$[this]
        endmethod

        method operator $NAME$= takes $TYPE$ arg returns nothing
            set fields[$ID$].$TYPE$[this] = arg
        endmethod
    //! endtextmacro
endlibrary

And a Zinc variant (because I use Zinc):
JASS:
library HashAlloc requires Table {
    //! textmacro SetupHashAllocStart takes COUNT
        private static Table recycler;
        private static TableArray fields;

        static method allocate() -> thistype {
            thistype this = recycler[0];

            if (!recycler.has(this)) {
                recycler[0] = this + 1;
            } else {
                recycler[0] = recycler[this];
            }

            return this;
        }

        method deallocate() {
            recycler[this] = recycler[0];
            recycler[0] = this;
        }

        private static method onInit() {
            recycler = Table.create();
            recycler[0] = 1;
            fields = TableArray[$COUNT$];
    //! endtextmacro

    //! textmacro SetupHashAllocEnd
        }
    //! endtextmacro

    //! textmacro SetupFieldThistype takes ID, NAME
        method operator $NAME$() -> thistype {
            return fields[$ID$].integer[this];
        }

        method operator $NAME$=(thistype arg) {
            fields[$ID$].integer[this] = arg;
        }
    //! endtextmacro


    //! textmacro SetupField takes ID, NAME, TYPE
        method operator $NAME$() -> $TYPE$ {
            return fields[$ID$].$TYPE$[this];
        }

        method operator $NAME$=($TYPE$ arg) {
            fields[$ID$].$TYPE$[this] = arg;
        }
    //! endtextmacro
}

Here's a usage example and test scenario demonstrating traversing a linked list of 16000 elements:
JASS:
library Test
    struct Test extends array
        //! runtextmacro SetupHashAllocStart("10")
        //! runtextmacro SetupHashAllocEnd()
      
        //! runtextmacro SetupFieldThistype("0", "prev")
        //! runtextmacro SetupField("1", "data", "string")
      
        static method create takes string value, thistype prev returns thistype
            local thistype this = allocate()
            set this.data = value
            set this.prev = prev
          
            return this
        endmethod
      
        method destroy takes nothing returns nothing
            call deallocate()
        endmethod
    endstruct
endlibrary

library TestCase initializer onInit requires Test
    globals
        private Test g_first
        private Test g_last
        private Test g_findFirst_ret
        private integer g_i
    endglobals

    function CreateInstances takes nothing returns nothing
        local integer i = 0
      
        // create 16384 instances and link them together
      
        loop
            set g_last = Test.create("instance #" + I2S(g_i), g_last)
            set i = i + 1
            set g_i = g_i + 1
            exitwhen i > 256 or g_i > 16384
        endloop
      
        if g_i < 16384 then
            call CreateInstances.execute()
        endif
    endfunction
  
    function FindFirst_aux takes Test node returns nothing
        local integer i = 0
      
        loop
            set node = node.prev
            set i = i + 1
            exitwhen i > 256 or node.prev == 0
        endloop
      
        if (node.prev != 0) then
            call FindFirst_aux.execute(node)
        else
            set g_findFirst_ret = node
        endif
    endfunction
  
    function FindFirst takes Test node returns Test
        call FindFirst_aux(node)
        return g_findFirst_ret
    endfunction
  
    private function onInit takes nothing returns nothing
        call BJDebugMsg("Setting up test")
  
        set g_i = 0
        set g_first = Test.create("first", 0)
        set g_last = g_first
      
        call BJDebugMsg("Creating instances")
        call CreateInstances.execute()
        call BJDebugMsg("First - " + g_first.data)
        call BJDebugMsg("Last - " + g_last.data)
        call BJDebugMsg("Traversed from last - " + FindFirst(g_last).data)
    endfunction
endlibrary

And here's a more complicated usage example from my map (albeit, in Zinc):
JASS:
public struct LoadQueue[] {
        //! runtextmacro SetupHashAllocStart("20")
        //! runtextmacro SetupHashAllocEnd()

        static thistype tops[]; // first in queue for each player
        static integer sizes[]; // queue sizes for each player

        // header

        //! runtextmacro SetupFieldThistype("0", "next")
        //! runtextmacro SetupFieldThistype("1", "prev")
        //! runtextmacro SetupField("2", "owner",      "integer")
        //! runtextmacro SetupField("3", "objectType", "integer")

        // shared

        //! runtextmacro SetupField("4", "id", "integer")
        //! runtextmacro SetupField("5", "x", "real")
        //! runtextmacro SetupField("6", "y", "real")

        // unit

        //! runtextmacro SetupField("7", "angle",      "real")
        //! runtextmacro SetupField("8", "size",       "real")
        //! runtextmacro SetupField("9", "speed",      "real")
        //! runtextmacro SetupField("10", "animSpeed", "real")
        //! runtextmacro SetupField("11", "z",         "real")
        //! runtextmacro SetupField("12", "color",     "integer")
        //! runtextmacro SetupField("13", "vertexR",   "integer")
        //! runtextmacro SetupField("14", "vertexG",   "integer")
        //! runtextmacro SetupField("15", "vertexB",   "integer")
        //! runtextmacro SetupField("16", "vertexA",   "integer")
        //! runtextmacro SetupField("17", "paused",    "boolean")
        //! runtextmacro SetupField("18", "tag",       "string")
        //! runtextmacro SetupField("19", "group",     "integer")

        // deform

        //! runtextmacro SetupField("7", "magnitude",  "real")
        //! runtextmacro SetupField("8", "radius",     "real")
}

Wouldn't it be nice if vJass supported an "extends hashtable" syntax for tables like these?
 
Level 13
Joined
Nov 7, 2014
Messages
571
Wouldn't it be nice if vJass supported an "extends hashtable" syntax for tables like these?
If your map has a lot of these structs that use more than 8190 instances, then yes, it would be a nice feature to have, but if your map has just a few of these then you could just "suck it up" and write the boilerplate which is pretter easy to do and saves some function calls.

JASS:
struct LoadQueue extends array
    private static hashtable ht = InitHashtable()
    private static thistype head = 0

    private static key k_next
    private method operator next takes nothing returns thistype
        return LoadInteger(ht, this, k_next)
    endmethod
    private method operator next= takes thistype other returns nothing
        call SaveInteger(ht, this, k_next, other)
    endmethod

    private static method allocate takes nothing returns thistype
        local thistype ms

        if head.next != 0 then
            set ms = head
            set head = head.next
        else
            set head = head + 1
            set ms = head
        endif

        set ms.next = 0

        return ms
    endmethod

    private method deallocate takes nothing returns nothing
        local LoadQueue ms = this
        if ms.next != 0 then
            call BJDebugMsg("|cffFF0000double free for instance LoadQueue(" + I2S(ms) + ")")
        endif
        call FlushChildHashtable(ht, ms)
        set ms.next = head
        set head = ms
    endmethod

    private static key k_prev
    private method operator prev takes nothing returns thistype
        return LoadInteger(ht, this, k_prev)
    endmethod
    private method operator prev= takes thistype other returns nothing
        call SaveInteger(ht, this, k_prev, other)
    endmethod

    private static key k_owner
    method operator owner takes nothing returns integer
        return LoadInteger(ht, this, k_owner)
    endmethod
    method operator owner= takes integer value returns nothing
        call SaveInteger(ht, this, k_owner, value)
    endmethod

    private static key k_objectType
    method operator objectType takes nothing returns integer
        return LoadInteger(ht, this, k_objectType)
    endmethod
    method operator objectType= takes integer value returns nothing
        call SaveInteger(ht, this, k_objectType, value)
    endmethod

    private static key k_id
    method operator id takes nothing returns integer
        return LoadInteger(ht, this, k_id)
    endmethod
    method operator id= takes integer value returns nothing
        call SaveInteger(ht, this, k_id, value)
    endmethod

    private static key k_x
    method operator x takes nothing returns real
        return LoadReal(ht, this, k_x)
    endmethod
    method operator x= takes integer value returns nothing
        call SaveReal(ht, this, k_x, value)
    endmethod

    private static key k_y
    method operator y takes nothing returns real
        return LoadReal(ht, this, k_y)
    endmethod
    method operator y= takes integer value returns nothing
        call SaveReal(ht, this, k_y, value)
    endmethod

    private static key k_z
    method operator z takes nothing returns real
        return LoadReal(ht, this, k_z)
    endmethod
    method operator z= takes integer value returns nothing
        call SaveReal(ht, this, k_z, value)
    endmethod

    private static key k_angle
    method operator angle takes nothing returns real
        return LoadReal(ht, this, k_angle)
    endmethod
    method operator angle= takes integer value returns nothing
        call SaveReal(ht, this, k_angle, value)
    endmethod

    private static key k_size
    method operator size takes nothing returns real
        return LoadReal(ht, this, k_size)
    endmethod
    method operator size= takes integer value returns nothing
        call SaveReal(ht, this, k_size, value)
    endmethod

    private static key k_speed
    method operator speed takes nothing returns real
        return LoadReal(ht, this, k_speed)
    endmethod
    method operator speed= takes integer value returns nothing
        call SaveReal(ht, this, k_speed, value)
    endmethod

    private static key k_animSpeed
    method operator animSpeed takes nothing returns real
        return LoadReal(ht, this, k_animSpeed)
    endmethod
    method operator animSpeed= takes integer value returns nothing
        call SaveReal(ht, this, k_animSpeed, value)
    endmethod

    private static key k_color
    method operator color takes nothing returns integer
        return LoadInteger(ht, this, k_color)
    endmethod
    method operator color= takes integer value returns nothing
        call SaveInteger(ht, this, k_color, value)
    endmethod

    private static key k_vertexR
    method operator vertexR takes nothing returns integer
        return LoadInteger(ht, this, k_vertexR)
    endmethod
    method operator vertexR= takes integer value returns nothing
        call SaveInteger(ht, this, k_vertexR, value)
    endmethod

    private static key k_vertexG
    method operator vertexG takes nothing returns integer
        return LoadInteger(ht, this, k_vertexG)
    endmethod
    method operator vertexG= takes integer value returns nothing
        call SaveInteger(ht, this, k_vertexG, value)
    endmethod

    private static key k_vertexB
    method operator vertexB takes nothing returns integer
        return LoadInteger(ht, this, k_vertexB)
    endmethod
    method operator vertexB= takes integer value returns nothing
        call SaveInteger(ht, this, k_vertexB, value)
    endmethod

    private static key k_vertexA
    method operator vertexA takes nothing returns integer
        return LoadInteger(ht, this, k_vertexA)
    endmethod
    method operator vertexA= takes integer value returns nothing
        call SaveInteger(ht, this, k_vertexA, value)
    endmethod

    private static key k_paused
    method operator paused takes nothing returns boolean
        return LoadBoolean(ht, this, k_paused)
    endmethod
    method operator paused= takes boolean value returns nothing
        call SaveBoolean(ht, this, k_paused, value)
    endmethod

    private static key k_tag
    method operator tag takes nothing returns string
        return LoadStr(ht, this, k_tag)
    endmethod
    method operator tag= takes string value returns nothing
        call SaveStr(ht, this, k_tag, value)
    endmethod

    private static key k_group
    method operator group takes nothing returns integer
        return LoadInteger(ht, this, k_group)
    endmethod
    method operator group= takes integer value returns nothing
        call SaveInteger(ht, this, k_group, value)
    endmethod


    static method create takes nothing returns LoadQueue
        return allocate()
    endmethod

    method destroy takes nothing returns nothing
        call this.deallocate()
    endmethod
endstruct
 

~El

Level 17
Joined
Jun 13, 2016
Messages
558
If your map has a lot of these structs that use more than 8190 instances, then yes, it would be a nice feature to have, but if your map has just a few of these then you could just "suck it up" and write the boilerplate which is pretter easy to do and saves some function calls.

JASS:
struct LoadQueue extends array
    private static hashtable ht = InitHashtable()
    private static thistype head = 0

    private static key k_next
    private method operator next takes nothing returns thistype
        return LoadInteger(ht, this, k_next)
    endmethod
    private method operator next= takes thistype other returns nothing
        call SaveInteger(ht, this, k_next, other)
    endmethod

    private static method allocate takes nothing returns thistype
        local thistype ms

        if head.next != 0 then
            set ms = head
            set head = head.next
        else
            set head = head + 1
            set ms = head
        endif

        set ms.next = 0

        return ms
    endmethod

    private method deallocate takes nothing returns nothing
        local LoadQueue ms = this
        if ms.next != 0 then
            call BJDebugMsg("|cffFF0000double free for instance LoadQueue(" + I2S(ms) + ")")
        endif
        call FlushChildHashtable(ht, ms)
        set ms.next = head
        set head = ms
    endmethod

    private static key k_prev
    private method operator prev takes nothing returns thistype
        return LoadInteger(ht, this, k_prev)
    endmethod
    private method operator prev= takes thistype other returns nothing
        call SaveInteger(ht, this, k_prev, other)
    endmethod

    private static key k_owner
    method operator owner takes nothing returns integer
        return LoadInteger(ht, this, k_owner)
    endmethod
    method operator owner= takes integer value returns nothing
        call SaveInteger(ht, this, k_owner, value)
    endmethod

    private static key k_objectType
    method operator objectType takes nothing returns integer
        return LoadInteger(ht, this, k_objectType)
    endmethod
    method operator objectType= takes integer value returns nothing
        call SaveInteger(ht, this, k_objectType, value)
    endmethod

    private static key k_id
    method operator id takes nothing returns integer
        return LoadInteger(ht, this, k_id)
    endmethod
    method operator id= takes integer value returns nothing
        call SaveInteger(ht, this, k_id, value)
    endmethod

    private static key k_x
    method operator x takes nothing returns real
        return LoadReal(ht, this, k_x)
    endmethod
    method operator x= takes integer value returns nothing
        call SaveReal(ht, this, k_x, value)
    endmethod

    private static key k_y
    method operator y takes nothing returns real
        return LoadReal(ht, this, k_y)
    endmethod
    method operator y= takes integer value returns nothing
        call SaveReal(ht, this, k_y, value)
    endmethod

    private static key k_z
    method operator z takes nothing returns real
        return LoadReal(ht, this, k_z)
    endmethod
    method operator z= takes integer value returns nothing
        call SaveReal(ht, this, k_z, value)
    endmethod

    private static key k_angle
    method operator angle takes nothing returns real
        return LoadReal(ht, this, k_angle)
    endmethod
    method operator angle= takes integer value returns nothing
        call SaveReal(ht, this, k_angle, value)
    endmethod

    private static key k_size
    method operator size takes nothing returns real
        return LoadReal(ht, this, k_size)
    endmethod
    method operator size= takes integer value returns nothing
        call SaveReal(ht, this, k_size, value)
    endmethod

    private static key k_speed
    method operator speed takes nothing returns real
        return LoadReal(ht, this, k_speed)
    endmethod
    method operator speed= takes integer value returns nothing
        call SaveReal(ht, this, k_speed, value)
    endmethod

    private static key k_animSpeed
    method operator animSpeed takes nothing returns real
        return LoadReal(ht, this, k_animSpeed)
    endmethod
    method operator animSpeed= takes integer value returns nothing
        call SaveReal(ht, this, k_animSpeed, value)
    endmethod

    private static key k_color
    method operator color takes nothing returns integer
        return LoadInteger(ht, this, k_color)
    endmethod
    method operator color= takes integer value returns nothing
        call SaveInteger(ht, this, k_color, value)
    endmethod

    private static key k_vertexR
    method operator vertexR takes nothing returns integer
        return LoadInteger(ht, this, k_vertexR)
    endmethod
    method operator vertexR= takes integer value returns nothing
        call SaveInteger(ht, this, k_vertexR, value)
    endmethod

    private static key k_vertexG
    method operator vertexG takes nothing returns integer
        return LoadInteger(ht, this, k_vertexG)
    endmethod
    method operator vertexG= takes integer value returns nothing
        call SaveInteger(ht, this, k_vertexG, value)
    endmethod

    private static key k_vertexB
    method operator vertexB takes nothing returns integer
        return LoadInteger(ht, this, k_vertexB)
    endmethod
    method operator vertexB= takes integer value returns nothing
        call SaveInteger(ht, this, k_vertexB, value)
    endmethod

    private static key k_vertexA
    method operator vertexA takes nothing returns integer
        return LoadInteger(ht, this, k_vertexA)
    endmethod
    method operator vertexA= takes integer value returns nothing
        call SaveInteger(ht, this, k_vertexA, value)
    endmethod

    private static key k_paused
    method operator paused takes nothing returns boolean
        return LoadBoolean(ht, this, k_paused)
    endmethod
    method operator paused= takes boolean value returns nothing
        call SaveBoolean(ht, this, k_paused, value)
    endmethod

    private static key k_tag
    method operator tag takes nothing returns string
        return LoadStr(ht, this, k_tag)
    endmethod
    method operator tag= takes string value returns nothing
        call SaveStr(ht, this, k_tag, value)
    endmethod

    private static key k_group
    method operator group takes nothing returns integer
        return LoadInteger(ht, this, k_group)
    endmethod
    method operator group= takes integer value returns nothing
        call SaveInteger(ht, this, k_group, value)
    endmethod


    static method create takes nothing returns LoadQueue
        return allocate()
    endmethod

    method destroy takes nothing returns nothing
        call this.deallocate()
    endmethod
endstruct

No doubt about that. It's a really specific corner case, but even in that example you've posted there's a lot of code that doesn't do much besides redefining some methods. It's tedious to write hashtable code that manages a lot of data, and it's further illustrated by the example you posted. And I like to avoid boilerplate whenever I can (I use a lot of textmacros in my map)

In my map right now I only have one struct like that, but I plan to convert some more to this style because I really need to manage a LOT of data in there.
One of the most useful uses for this would probably be a Unit struct like one from AIDS, but without the 8191 limit. Yes, it is unlikely that you'll ever hit this limit in most maps, but even vJass supports (natively) structs above 8191 instances using an ID selector and multiple arrays (which arguably has an even bigger overhead than hashtables)

Hence, my rather innocent desire to have this supported in JassHelper. Heck, I'd even go and implement it myself if I had the time for it, but alas I'm not very experienced with compilers and parsers.
 

~El

Level 17
Joined
Jun 13, 2016
Messages
558
Maybe im missing something but what about this

It actually generates a "switcher" method for each field access, which is... well, not very nice. Not to mention default vJass structs have a lot of issues with their default allocation implementation.

This simple example:
JASS:
library Test2
    struct SelectorSpam[80000]
        integer a
    endstruct
  
    function onInit takes nothing returns nothing
        local SelectorSpam s = SelectorSpam.create()
        set s.a = 10
    endfunction
endlibrary

Generates code like this for field 'a':
JASS:
function sg__SelectorSpam_get_a takes integer i returns integer
    if(i<8191) then
        return s__SelectorSpam_a[i]
    elseif(i<40955) then
        if(i<16382) then
            return s__SelectorSpam_2a[i-8191]
        elseif(i<24573) then
                return s__SelectorSpam_3a[i-16382]
        elseif(i<32764) then
            return s__SelectorSpam_4a[i-24573]
        else
            return s__SelectorSpam_5a[i-32764]
        endif
    elseif(i<49146) then
        return s__SelectorSpam_6a[i-40955]
    elseif(i<65528) then
        if(i<57337) then
            return s__SelectorSpam_7a[i-49146]
        else
            return s__SelectorSpam_8a[i-57337]
        endif
    elseif(i<73719) then
        return s__SelectorSpam_9a[i-65528]
    else
        return s__SelectorSpam_10a[i-73719]
    endif
endfunction

function sg__SelectorSpam_set_a takes integer i,integer v returns nothing
    if(i<8191) then
        set s__SelectorSpam_a[i]=v
    elseif(i<40955) then
        if(i<16382) then
            set s__SelectorSpam_2a[i-8191]=v
        elseif(i<24573) then
                set s__SelectorSpam_3a[i-16382]=v
        elseif(i<32764) then
            set s__SelectorSpam_4a[i-24573]=v
        else
            set s__SelectorSpam_5a[i-32764]=v
        endif
    elseif(i<49146) then
        set s__SelectorSpam_6a[i-40955]=v
    elseif(i<65528) then
        if(i<57337) then
            set s__SelectorSpam_7a[i-49146]=v
        else
            set s__SelectorSpam_8a[i-57337]=v
        endif
    elseif(i<73719) then
        set s__SelectorSpam_9a[i-65528]=v
    else
        set s__SelectorSpam_10a[i-73719]=v
    endif
endfunction

And they are still limited in size. With enormous struct sizes, the selector function really grows.

EDIT: And these are the arrays it generates:
JASS:
integer array si__SelectorSpam_V
integer array si__SelectorSpam_2V
integer array si__SelectorSpam_3V
integer array si__SelectorSpam_4V
integer array si__SelectorSpam_5V
integer array si__SelectorSpam_6V
integer array si__SelectorSpam_7V
integer array si__SelectorSpam_8V
integer array si__SelectorSpam_9V
integer array si__SelectorSpam_10V
integer array s__SelectorSpam_2a
integer array s__SelectorSpam_3a
integer array s__SelectorSpam_4a
integer array s__SelectorSpam_5a
integer array s__SelectorSpam_6a
integer array s__SelectorSpam_7a
integer array s__SelectorSpam_8a
integer array s__SelectorSpam_9a
integer array s__SelectorSpam_10a
integer array s__SelectorSpam_a

Compared to only a single hashtable in the approach I offered.
 
Status
Not open for further replies.
Top