Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
solvers-acuk-m
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Takhir Fakhrutdinov
solvers-acuk-m
Commits
d24f1859
Commit
d24f1859
authored
Oct 07, 2022
by
Takhir Fakhrutdinov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Обработка файлов просмотренных зон
parent
54c7c2c8
Changes
4
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
70 additions
and
36 deletions
+70
-36
frames_20221005_030409_3.json
fte/lib/libpy/data/frames_20221005_030409_3.json
+1
-1
fileFactory.py
fte/lib/libpy/fileFactory.py
+2
-1
lczFile.py
fte/lib/libpy/lczFile.py
+9
-8
test_factory.py
fte/loader/test/test_factory.py
+58
-26
No files found.
fte/lib/libpy/data/frames_20221005_030409_3.json
View file @
d24f1859
This diff is collapsed.
Click to expand it.
fte/lib/libpy/fileFactory.py
View file @
d24f1859
...
...
@@ -81,7 +81,7 @@ class Factory_t:
for
l
in
fd
:
l
=
l
.
strip
(
'
\n
'
)
.
strip
()
if
not
len
(
l
):
continue
if
l
[
0
]
==
'{'
:
if
l
[
0
]
in
(
'{'
,
'['
)
:
return
self
.
_detect_json
(
fd
)
else
:
for
k
,
v
in
self
.
_patterns
.
items
():
...
...
@@ -411,3 +411,4 @@ frame = {
}
}
factory
.
register_file
(
'LCZ'
,
_create_lczFile
,
schema
=
frame
)
fte/lib/libpy/lczFile.py
View file @
d24f1859
...
...
@@ -3,6 +3,7 @@ from .xFile import baseFile
import
logging
import
json
import
io
from
datetime
import
datetime
logger
=
logging
.
getLogger
(
''
)
...
...
@@ -25,7 +26,6 @@ class lczFile(baseFile):
async
def
process_data
(
self
,
data
):
import
base64
from
datetime
import
datetime
from
collections
import
namedtuple
real
=
lambda
x
:
round
(
float
(
x
),
6
)
...
...
@@ -44,20 +44,21 @@ class lczFile(baseFile):
_
=
{
k
:
f
(
r
[
k
])
for
k
,
f
in
rec_list
.
items
()
if
k
in
r
}
rec
=
namedtuple
(
'rec_t'
,
_
.
keys
())(
**
_
)
crops
=
r'\x
%
s'
%
base64
.
b64decode
(
rec
.
crops
)
.
hex
()
crops
=
base64
.
b64decode
(
rec
.
crops
)
if
rec
.
night
not
in
nights
:
await
self
.
execute
(
"select * from template.partition_table('xd.t_localzone'::regclass,$1)"
,
rec
.
night
)
nights
.
add
(
rec
.
night
)
if
rec
.
night
id
not
in
nights
:
await
self
.
execute
(
"select * from template.partition_table('xd.t_localzone'::regclass,$1)"
,
rec
.
night
id
)
nights
.
add
(
rec
.
night
id
)
dev
=
await
self
.
fetchval
(
"""
insert into xd.t_localzone(night,dev,fn,fid,tm,frame,crops)
values ($1,main.devget_
devid($2,$1::date),$3,$4,$5,$6,$7
)
values ($1,main.devget_
id($2::integer,$3),$4,$5,$6,$7,$8
)
on conflict(dev,night,fn) do update
set (fid,tm,frame,crops) = (excluded.fid,excluded.tm,excluded.frame,excluded.crops)
returning dev
"""
,
rec
.
night
,
rec
.
siteid
,
rec
.
filename
,
self
.
fid
,
rec
.
osb
time
,
json
.
dumps
(
frame
),
crops
)
"""
,
rec
.
night
id
,
rec
.
siteid
,
rec
.
nightid
.
date
(),
rec
.
filename
,
self
.
fid
,
rec
.
obs
time
,
json
.
dumps
(
frame
),
crops
)
devs
.
add
(
dev
)
tb
,
te
=
m
ax
(
tb
,
rec
.
osbtime
),
min
(
te
,
rec
.
osb
time
)
tb
,
te
=
m
in
(
tb
,
rec
.
obstime
),
max
(
te
,
rec
.
obs
time
)
self
.
devs
,
self
.
tbeg
,
self
.
tend
=
list
(
devs
),
tb
,
te
...
...
fte/loader/test/test_factory.py
View file @
d24f1859
...
...
@@ -49,35 +49,67 @@ if __name__ == '__main__':
t
=
factory
.
detect_file
(
f
)
if
not
t
:
shema
=
{
"type"
:
"array"
,
"items"
:
{
"allOf"
:
[
{
"type"
:
"object"
,
"required"
:
[
"header"
,
"frames"
],
"properties"
:
{
"tracks"
:
{
"type"
:
"array"
,
"items"
:{
"$ref"
:
"#/$defs/track"
}},
"siteid"
:
{
"type"
:
"integer"
},
"nightid"
:
{
"type"
:
"integer"
}
},
"required"
:
[
"siteid"
,
"nightid"
,
"tracks"
],
"$defs"
:{
"track"
:{
"header"
:
{
"type"
:
"object"
,
"required"
:
[
"originator"
,
"message_type"
,
"message_id"
],
"properties"
:
{
"trackid"
:
{
"type"
:
"integer"
},
"meas"
:
{
"type"
:
"array"
,
"items"
:{
"$ref"
:
"#/$defs/ms"
}},
},
"required"
:
[
"trackid"
,
"meas"
],
},
"ms"
:{
"properties"
:{
"utc"
:{
"type"
:
"string"
,
"pattern"
:
r"\d{2}\/\d{2}\/\d{4}\s\d{2}\:\d{2}\:\d{2}(\.\d+)?"
},
"ra_j2000"
:{
"type"
:
"string"
,
"pattern"
:
r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?"
},
"dec_j2000"
:{
"type"
:
"string"
,
"pattern"
:
r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?"
},
"mag"
:{
"type"
:
"string"
,
"pattern"
:
r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?"
},
"ra_j2000_full_error"
:{
"type"
:
"string"
,
"pattern"
:
r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?"
},
"dec_j2000_full_error"
:{
"type"
:
"string"
,
"pattern"
:
r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?"
},
"mag_error"
:{
"type"
:
"string"
,
"pattern"
:
r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?"
},
"crops"
:{
"type"
:
"string"
,
"media"
:{
"binaryEncoding"
:
"base64"
,
"type"
:
"image/png"
}}
"originator"
:
{
"type"
:
"string"
},
"message_type"
:
{
"type"
:
"string"
,
"enum"
:[
"frames"
]},
"message_id"
:
{
"type"
:
"string"
}
}
},
"required"
:
[
"utc"
,
"ra_j2000"
,
"dec_j2000"
,
"mag"
,
"ra_j2000_full_error"
,
"dec_j2000_full_error"
,
"mag_error"
,
"crops"
]
"frames"
:
{
"type"
:
"array"
,
"items"
:
{
"allOf"
:[
{
"type"
:
"object"
,
"required"
:
[
"siteid"
,
"nightid"
,
"filename"
,
"exp_num"
,
"obstime"
,
"crops"
,
"ast_ra_j2000"
,
"ast_dec_j2000"
,
"ast_rms_ra"
,
"ast_rms_dec"
,
"ast_orientation"
,
"limmag"
,
"limmag_error"
,
"phot_zero_error"
,
"phot_zero"
,
"sky_level"
,
"seeing"
],
"properties"
:
{
"siteid"
:
{
"$ref"
:
"#/$defs/integer"
},
"nightid"
:
{
"$ref"
:
"#/$defs/integer"
},
"filename"
:
{
"type"
:
"string"
},
"exp_num"
:
{
"type"
:
"integer"
},
"obstime"
:
{
"$ref"
:
"#/$defs/datetime"
},
"crops"
:
{
"$ref"
:
"#/$defs/base64"
},
"ast_ra_j2000"
:
{
"$ref"
:
"#/$defs/number"
},
"ast_dec_j2000"
:
{
"$ref"
:
"#/$defs/number"
},
"ast_rms_ra"
:
{
"$ref"
:
"#/$defs/number"
},
"ast_rms_dec"
:
{
"$ref"
:
"#/$defs/number"
},
"ast_orientation"
:
{
"$ref"
:
"#/$defs/number"
},
"limmag"
:
{
"$ref"
:
"#/$defs/number"
},
"limmag_error"
:
{
"$ref"
:
"#/$defs/number"
},
"phot_zero_error"
:
{
"$ref"
:
"#/$defs/number"
},
"phot_zero"
:
{
"$ref"
:
"#/$defs/number"
},
"sky_level"
:
{
"$ref"
:
"#/$defs/number"
},
"seeing"
:
{
"$ref"
:
"#/$defs/number"
}
}
}
]
}
}
}
}
]
},
"$defs"
:
{
"datetime"
:
{
"type"
:
"string"
,
"pattern"
:
"^
\\
d{2}
\\
/
\\
d{2}
\\
/
\\
d{4}
\\
s
\\
d{2}
\\
:
\\
d{2}
\\
:
\\
d{2}(
\\
.
\\
d+)?$"
},
"integer"
:
{
"type"
:
"string"
,
"pattern"
:
"^[-+]?
\\
d+$"
},
"number"
:
{
"type"
:
"string"
,
"pattern"
:
"^[-+]?[0-9]*
\\
.?[0-9]+([eE][-+]?[0-9]+)?$"
},
"base64"
:
{
"type"
:
"string"
,
"pattern"
:
"^(?=(.{4})*$)[A-Za-z0-9+
\\
/]*={0,2}$"
}
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment