diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml new file mode 100644 index 00000000..4e214f30 --- /dev/null +++ b/.github/workflows/backup.yml @@ -0,0 +1,45 @@ +name: Mirror repo to S3 + +on: + schedule: + # Runs everyday at 08:38 am + - cron: '38 8 * * *' + workflow_dispatch: + +jobs: + s3Backup: + runs-on: docker-builds + steps: + - uses: actions/checkout@v1 + + - name: Get info + id: get_info + run: | + GIT_COMMIT_DATE="$((`git log -n 1 --date-order --all | grep Date | awk '{ print $4 }'`))" + YESTERDAY_DATE="$((`date | awk '{ print $3 }'`-1))" + echo ::set-output name=GIT_COMMIT_DATE::${GIT_COMMIT_DATE} + echo ::set-output name=YESTERDAY_DATE::${YESTERDAY_DATE} + echo $GIT_COMMIT_DATE + echo $YESTERDAY_DATE + + - name: Create backup + if: steps.get_info.outputs.GIT_COMMIT_DATE == steps.get_info.outputs.YESTERDAY_DATE + run: | + github-backup -O 0chain -P -t ${{ secrets.ACCESS_TOKEN }} --output-directory=/github-backup/msgp --all -R msgp + + - name: Create zip + if: steps.get_info.outputs.GIT_COMMIT_DATE == steps.get_info.outputs.YESTERDAY_DATE + run: zip -r msgp.zip /github-backup/msgp + + - name: Set AWS credentials + if: steps.get_info.outputs.GIT_COMMIT_DATE == steps.get_info.outputs.YESTERDAY_DATE + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} + aws-region: us-east-2 + + - name: Backup to s3 + if: steps.get_info.outputs.GIT_COMMIT_DATE == steps.get_info.outputs.YESTERDAY_DATE + run: | + aws s3 cp msgp.zip s3://${{ secrets.MIRROR_TARGET }}/msgp.zip diff --git a/gen/elem.go b/gen/elem.go index 87a88d32..0d4d0398 100644 --- a/gen/elem.go +++ b/gen/elem.go @@ -88,9 +88,10 @@ const ( Int32 Int64 Bool - Intf // interface{} - Time // time.Time - Ext // extension + Intf // interface{} + Time // time.Time + Duration // time.Duration + Ext // extension IDENT // IDENT means an unrecognized identifier ) @@ -119,6 +120,7 @@ var primitives = map[string]Primitive{ "bool": Bool, "interface{}": Intf, "time.Time": Time, + "time.Duration": Duration, "msgp.Extension": Ext, } @@ -578,6 +580,11 @@ func (s *BaseElem) BaseName() string { if s.Value == Time { return "Time" } + + if s.Value == Duration { + return "Duration" + } + return s.Value.String() } @@ -594,6 +601,8 @@ func (s *BaseElem) BaseType() string { return "[]byte" case Time: return "time.Time" + case Duration: + return "time.Duration" case Ext: return "msgp.Extension" @@ -665,6 +674,8 @@ func (s *BaseElem) ZeroExpr() string { case Time: return "(time.Time{})" + case Duration: + return "(time.Duration{})" } return "" @@ -721,6 +732,8 @@ func (k Primitive) String() string { return "Intf" case Time: return "time.Time" + case Duration: + return "time.Duration" case Ext: return "Extension" case IDENT: diff --git a/gen/marshal.go b/gen/marshal.go index fc5c5930..eaffc101 100644 --- a/gen/marshal.go +++ b/gen/marshal.go @@ -221,9 +221,19 @@ func (m *marshalGen) gMap(s *Map) { m.fuseHook() vname := s.Varname() m.rawAppend(mapHeader, lenAsUint32, vname) - m.p.printf("\nfor %s, %s := range %s {", s.Keyidx, s.Validx, vname) - m.rawAppend(stringTyp, literalFmt, s.Keyidx) - m.ctx.PushVar(s.Keyidx) + + keys := fmt.Sprintf("keys_%s", s.Keyidx) + m.p.printf("\n%s := make([]string, 0, len(%s))", keys, vname) + m.p.printf("\nfor k := range %s {", vname) + m.p.printf("\n%s = append(%s, k)", keys, keys) + m.p.printf("\n}") + m.p.printf("\nmsgp.Sort(%s)", keys) + + m.p.printf("\nfor _, k := range %s {", keys) + m.p.printf("\n%s := %s[k]", s.Validx, vname) + m.rawAppend(stringTyp, "%s", "k") + + m.ctx.PushVar("k") next(m, s.Value) m.ctx.Pop() m.p.closeblock() diff --git a/msgp/json_bytes.go b/msgp/json_bytes.go index 438caf53..990dacf7 100644 --- a/msgp/json_bytes.go +++ b/msgp/json_bytes.go @@ -16,14 +16,15 @@ func init() { // NOTE(pmh): this is best expressed as a jump table, // but gc doesn't do that yet. revisit post-go1.5. unfuns = [_maxtype]func(jsWriter, []byte, []byte) ([]byte, []byte, error){ - StrType: rwStringBytes, - BinType: rwBytesBytes, - MapType: rwMapBytes, - ArrayType: rwArrayBytes, - Float64Type: rwFloat64Bytes, - Float32Type: rwFloat32Bytes, - BoolType: rwBoolBytes, - IntType: rwIntBytes, + StrType: rwStringBytes, + BinType: rwBytesBytes, + MapType: rwMapBytes, + ArrayType: rwArrayBytes, + Float64Type: rwFloat64Bytes, + Float32Type: rwFloat32Bytes, + BoolType: rwBoolBytes, + IntType: rwIntBytes, + // DurationType: rwIntBytes, UintType: rwUintBytes, NilType: rwNullBytes, ExtensionType: rwExtensionBytes, diff --git a/msgp/read.go b/msgp/read.go index e2b63c2b..f5a8aa0c 100644 --- a/msgp/read.go +++ b/msgp/read.go @@ -44,6 +44,7 @@ const ( Complex64Type Complex128Type TimeType + // DurationType _maxtype ) @@ -1259,6 +1260,16 @@ func (m *Reader) ReadTime() (t time.Time, err error) { return } +func (m *Reader) ReadDuration() (d time.Duration, err error) { + var i int64 + i, err = m.ReadInt64() + if err != nil { + return + } + + return time.Duration(i), nil +} + // ReadIntf reads out the next object as a raw interface{}. // Arrays are decoded as []interface{}, and maps are decoded // as map[string]interface{}. Integers are decoded as int64 diff --git a/msgp/read_bytes.go b/msgp/read_bytes.go index 2e575454..8540dac6 100644 --- a/msgp/read_bytes.go +++ b/msgp/read_bytes.go @@ -1027,6 +1027,13 @@ func ReadTimeBytes(b []byte) (t time.Time, o []byte, err error) { return } +func ReadDurationBytes(b []byte) (d time.Duration, o []byte, err error) { + var v int64 + v, o, err = ReadInt64Bytes(b) + d = time.Duration(v) + return +} + // ReadMapStrIntfBytes reads a map[string]interface{} // out of 'b' and returns the map and remaining bytes. // If 'old' is non-nil, the values will be read into that map. @@ -1124,6 +1131,12 @@ func ReadIntfBytes(b []byte) (i interface{}, o []byte, err error) { i, o, err = ReadTimeBytes(b) return + // case DurationType: + // var v int64 + // v, o, err = ReadInt64Bytes(b) + // i = time.Duration(v) + // return + case Complex64Type: i, o, err = ReadComplex64Bytes(b) return diff --git a/msgp/size.go b/msgp/size.go index ce2f8b16..029d0774 100644 --- a/msgp/size.go +++ b/msgp/size.go @@ -25,9 +25,10 @@ const ( Complex64Size = 10 Complex128Size = 18 - TimeSize = 15 - BoolSize = 1 - NilSize = 1 + TimeSize = 15 + DurationSize = Int64Size + BoolSize = 1 + NilSize = 1 MapHeaderSize = 5 ArrayHeaderSize = 5 diff --git a/msgp/write.go b/msgp/write.go index f6ad9a05..b308c779 100644 --- a/msgp/write.go +++ b/msgp/write.go @@ -5,6 +5,7 @@ import ( "io" "math" "reflect" + "sort" "sync" "time" ) @@ -619,6 +620,10 @@ func (mw *Writer) WriteTime(t time.Time) error { return nil } +func (mw *Writer) WriteDuration(t time.Duration) error { + return mw.WriteInt64(int64(t)) +} + // WriteIntf writes the concrete type of 'v'. // WriteIntf will error if 'v' is not one of the following: // - A bool, float, string, []byte, int, uint, or complex @@ -682,6 +687,8 @@ func (mw *Writer) WriteIntf(v interface{}) error { return mw.WriteMapStrIntf(v) case time.Time: return mw.WriteTime(v) + case time.Duration: + return mw.WriteDuration(v) } val := reflect.ValueOf(v) @@ -858,3 +865,7 @@ func GuessSize(i interface{}) int { return 512 } } + +func Sort(vs []string) { + sort.Strings(vs) +} diff --git a/msgp/write_bytes.go b/msgp/write_bytes.go index 93d6d764..cdcbca03 100644 --- a/msgp/write_bytes.go +++ b/msgp/write_bytes.go @@ -305,6 +305,10 @@ func AppendTime(b []byte, t time.Time) []byte { return o } +func AppendDuration(b []byte, t time.Duration) []byte { + return AppendInt64(b, int64(t)) +} + // AppendMapStrStr appends a map[string]string to the slice // as a MessagePack map with 'str'-type keys and values func AppendMapStrStr(b []byte, m map[string]string) []byte { @@ -390,6 +394,8 @@ func AppendIntf(b []byte, i interface{}) ([]byte, error) { return AppendUint64(b, i), nil case time.Time: return AppendTime(b, i), nil + case time.Duration: + return AppendDuration(b, i), nil case map[string]interface{}: return AppendMapStrIntf(b, i) case map[string]string: