We finished a long time after you went offline, so to not make you read all the tests we made, here is the final one:
function GetNearestUnitInGroupFromXY takes unit U, real x1, real y1, group g returns unit
local unit u=FirstOfGroup(g)
local real x2=GetUnitX(u)-x1
local real y2=GetUnitY(u)-y1
local real dist1=SquareRoot(x2*x2 + y2*y2)
local real dist2
set U=u
call GroupRemoveUnit(g,u)
loop
set u=FirstOfGroup(g)
exitwhen u==null
set x2=GetUnitX(u)-x1
set y2=GetUnitY(u)-y1
set dist2=SquareRoot(x2*x2 + y2*y2)
if (dist1>dist2) then
set dist1=dist2
set U=u
endif
call GroupRemoveUnit(g,u)
endloop
set u=null
return U
endfunction
//Example Usage:
function Call takes group g returns nothing
local unit U
local real x = 0
local real y = 0
set U = GetNearestUnitInGroupFromXY(U,x,y,g)
endfunction