在Go中從Python項目加載數據存儲實體導致嵌套的結構片的切片錯誤

Question

由於性能方面的原因，我在Google的Google AppEngine項目中編寫了一個模塊，但需要能夠讀取數據存儲區中的某些實體。我寫出來的Go代碼，以便能夠讀取我在Python建出來的實體，但我收到以下錯誤：

datastore: flattening nested structs leads to a slice of slices: field "Messages"

模型定義在Python：

class ModelB(ndb.Model): 
    msg_id = ndb.StringProperty(indexed=False) 
    cat_ids = ndb.StringProperty(repeated=True, indexed=False) 
    list_ids = ndb.StringProperty(repeated=True, indexed=False) 
    default_list_id_index = ndb.IntegerProperty(indexed=False) 

class ModelA(ndb.Model): 
    date_join = ndb.DateTimeProperty(auto_now_add=True) 
    name = ndb.StringProperty() 
    owner_salutation = ndb.StringProperty(indexed=False) 
    owner_email_address = ndb.StringProperty() 
    logo_url = ndb.StringProperty(indexed=False) 
    ... 
    messages = ndb.LocalStructuredProperty(ModelB, name='bm', repeated=True) 

而且在圍棋：

type ModelB struct { 
    MessageID   string `datastore:"msg_id,noindex"` 
    CategoryIDs  []string `datastore:"cat_ids,noindex"` 
    ListIDs   []string `datastore:"list_ids,noindex"` 
    DefaultListIDIndex int  `datastore:"default_list_id_index,noindex"` 
} 

type ModelA struct { 
    DateJoin   time.Time `datastore:"date_join,"` 
    Name    string `datastore:"name,"` 
    OwnerSalutation string `datastore:"owner_salutation,noindex"` 
    OwnerEmailAddress string `datastore:"owner_email_address,"` 
    LogoURL   string `datastore:"logo_url,noindex"` 
    Messages   []ModelB `datastore:"bm,"` 
} 

有什麼我在這裏做錯了嗎？只是Go與Python模型定義之間的功能不兼容？

試圖解碼ModelB

重新定義ModelA如下：

import pb "appengine_internal/datastore" 
import proto "code.google.com/p/goprotobuf/proto" 

type ModelA struct { 
    DateJoin   time.Time `datastore:"date_join,"` 
    Name    string `datastore:"name,"` 
    OwnerSalutation string `datastore:"owner_salutation,noindex"` 
    OwnerEmailAddress string `datastore:"owner_email_address,"` 
    LogoURL   string `datastore:"logo_url,noindex"` 
    Messages   []ModelB `datastore:"-"` 
} 

// Load is implemented for the PropertyLoaderSaver interface. 

func (seller *ModelA) Load(c <-chan datastore.Property) error { 
    f := make(chan datastore.Property, 100) 
    for p := range c { 
    if p.Name == "bm" { 
     var val pb.EntityProto 
     err := proto.Unmarshal([]byte(p.Value.(string)), &val) 
     if err != nil { 
     return err 
     } 
     //TODO: Store result as a new ModelB 
    } else { 
     f <- p 
    } 
    } 
    close(f) 
    return datastore.LoadStruct(seller, f) 
} 

但我收到以下錯誤： proto: required field "{Unknown}" not set

Answer 1

我想，如果你挖夠了，你會找到答案：

首先，在Python定義LocalStructuredProperty屬性時，需要設置keep_keys=True

class ModelB(ndb.Model): 
    msg_id = ndb.StringProperty(indexed=False) 
    cat_ids = ndb.StringProperty(repeated=True, indexed=False) 
    list_ids = ndb.StringProperty(repeated=True, indexed=False) 
    default_list_id_index = ndb.IntegerProperty(indexed=False) 

class ModelA(ndb.Model): 
    date_join = ndb.DateTimeProperty(auto_now_add=True) 
    name = ndb.StringProperty() 
    owner_salutation = ndb.StringProperty(indexed=False) 
    owner_email_address = ndb.StringProperty() 
    logo_url = ndb.StringProperty(indexed=False) 
    ... 
    messages = ndb.LocalStructuredProperty(ModelB, name='bm', repeated=True, keep_keys=True) 

在我的代碼和映射在我的實體簡單的重新定義做每個搞掂表示的put()。

然後在我的Go代碼：

type ModelB struct { 
    MessageID   string `datastore:"msg_id,noindex"` 
    CategoryIDs  []string `datastore:"cat_ids,noindex"` 
    ListIDs   []string `datastore:"list_ids,noindex"` 
    DefaultListIDIndex int  `datastore:"default_list_id_index,noindex"` 
} 

type ModelA struct { 
    DateJoin   time.Time `datastore:"date_join,"` 
    Name    string `datastore:"name,"` 
    OwnerSalutation string `datastore:"owner_salutation,noindex"` 
    OwnerEmailAddress string `datastore:"owner_email_address,"` 
    LogoURL   string `datastore:"logo_url,noindex"` 
    Messages   []ModelB `datastore:"-"` 
} 

// Load is implemented for the PropertyLoaderSaver interface. 
func (s *ModelA) Load(c <-chan datastore.Property) (err error) { 
    f := make(chan datastore.Property, 32) 
    errc := make(chan error, 1) 
    defer func() { 
     if err == nil { 
      err = <-errc 
     } 
    }() 
    go func() { 
     defer close(f) 
     for p := range c { 
      if p.Name == "bm" { 
       var b ModelB 
       err := loadLocalStructuredProperty(&b, []byte(p.Value.(string))) 
       if err != nil { 
        errc <- err 
        return 
       } 
       s.Messages = append(s.Messages, b) 
      } else { 
       f <- p 
      } 
     } 
     errc <- nil 
    }() 
    return datastore.LoadStruct(s, f) 
} 

我有一堆從appengine/datastore包複製的一個關鍵功能不被導出，並簡化我需要複製的代碼量，我放棄了支持爲Reference類型。我打開的問題跟蹤一票，看看我們是否能得到loadEntity功能輸出：https://code.google.com/p/googleappengine/issues/detail?id=10426

import ( 
    "errors"  
    "time"  

    "appengine"  
    "appengine/datastore"   

    pb "appengine_internal/datastore"  
    proto "code.google.com/p/goprotobuf/proto"  
)  

func loadLocalStructuredProperty(dst interface{}, raw_proto []byte) error {  
    var val pb.EntityProto  
    err := proto.Unmarshal(raw_proto, &val)  
    if err != nil {  
     return err  
    }  
    return loadEntity(dst, &val)  
} 

//Copied from appengine/datastore since its not exported 

// loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer. 
func loadEntity(dst interface{}, src *pb.EntityProto) (err error) { 
c := make(chan datastore.Property, 32) 
errc := make(chan error, 1) 
defer func() { 
    if err == nil { 
      err = <-errc 
     } 
    }() 
    go protoToProperties(c, errc, src) 
    if e, ok := dst.(datastore.PropertyLoadSaver); ok { 
     return e.Load(c) 
    } 
    return datastore.LoadStruct(dst, c) 
} 

func protoToProperties(dst chan<- datastore.Property, errc chan<- error, src *pb.EntityProto) { 
    defer close(dst) 
    props, rawProps := src.Property, src.RawProperty 
    for { 
     var (
      x  *pb.Property 
      noIndex bool 
     ) 
     if len(props) > 0 { 
      x, props = props[0], props[1:] 
     } else if len(rawProps) > 0 { 
      x, rawProps = rawProps[0], rawProps[1:] 
      noIndex = true 
     } else { 
      break 
     } 

     var value interface{} 
     if x.Meaning != nil && *x.Meaning == pb.Property_INDEX_VALUE { 
      value = indexValue{x.Value} 
     } else { 
      var err error 
      value, err = propValue(x.Value, x.GetMeaning()) 
      if err != nil { 
       errc <- err 
       return 
      } 
     } 
     dst <- datastore.Property{ 
      Name:  x.GetName(), 
      Value: value, 
      NoIndex: noIndex, 
      Multiple: x.GetMultiple(), 
     } 
    } 
    errc <- nil 
} 

func fromUnixMicro(t int64) time.Time { 
    return time.Unix(t/1e6, (t%1e6)*1e3) 
} 

// propValue returns a Go value that combines the raw PropertyValue with a 
// meaning. For example, an Int64Value with GD_WHEN becomes a time.Time. 
func propValue(v *pb.PropertyValue, m pb.Property_Meaning) (interface{}, error) { 
    switch { 
    case v.Int64Value != nil: 
     if m == pb.Property_GD_WHEN { 
      return fromUnixMicro(*v.Int64Value), nil 
     } else { 
      return *v.Int64Value, nil 
     } 
    case v.BooleanValue != nil: 
     return *v.BooleanValue, nil 
    case v.StringValue != nil: 
     if m == pb.Property_BLOB { 
      return []byte(*v.StringValue), nil 
     } else if m == pb.Property_BLOBKEY { 
      return appengine.BlobKey(*v.StringValue), nil 
     } else { 
      return *v.StringValue, nil 
     } 
    case v.DoubleValue != nil: 
     return *v.DoubleValue, nil 
    case v.Referencevalue != nil: 
     return nil, errors.New("Not Implemented!") 
    } 
    return nil, nil 
} 

// indexValue is a Property value that is created when entities are loaded from 
// an index, such as from a projection query. 
// 
// Such Property values do not contain all of the metadata required to be 
// faithfully represented as a Go value, and are instead represented as an 
// opaque indexValue. Load the properties into a concrete struct type (e.g. by 
// passing a struct pointer to Iterator.Next) to reconstruct actual Go values 
// of type int, string, time.Time, etc. 
type indexValue struct { 
    value *pb.PropertyValue 
} 

Answer 2

轉到數據存儲包不支持兩個像這樣的切片層。您可以有[]ModelB，只要ModelB不包含任何切片。或者，您可以在ModelA中使用ModelB，而ModelB可以在其中包含切片。但是你不能同時擁有[]ModelB和ModelB有切片。有關錯誤情況，請參閱the code。您的選擇：

1. 不這樣做，在圍棋
2. 寫自己的數據存儲解串器來處理這種情況 - 這可能是很難
3. 改變你的Python數據結構，以滿足圍棋要求和重寫數據

Answer 3

通過someone1解決方案的工作很好，但我有幾百萬的實體，並沒有想得重新把它們全部（把keep_keys = True添加到LocalStructuredProperty）。

所以，我創建的EntityProto一個刪節版本，刪除鍵&路徑等的依賴......這簡直是用LocalEntityProto取代pb.EntityProto和現有的Python編寫的實體應該加載OK（我使用PropertyLoadSaver爲嵌套結構）。

聲明：我只使用它從Go中讀取 - 我還沒有嘗試寫回相同的實體以查看它們是否仍然在Python中加載。

import pb "google.golang.org/appengine/internal/datastore" 
import proto "github.com/golang/protobuf/proto" 

type LocalEntityProto struct { 
    Kind    *pb.EntityProto_Kind `protobuf:"varint,4,opt,name=kind,enum=appengine.EntityProto_Kind" json:"kind,omitempty"` 
    KindUri   *string    `protobuf:"bytes,5,opt,name=kind_uri" json:"kind_uri,omitempty"` 
    Property   []*pb.Property  `protobuf:"bytes,14,rep,name=property" json:"property,omitempty"` 
    RawProperty  []*pb.Property  `protobuf:"bytes,15,rep,name=raw_property" json:"raw_property,omitempty"` 
    Rank    *int32    `protobuf:"varint,18,opt,name=rank" json:"rank,omitempty"` 
    XXX_unrecognized []byte    `json:"-"` 
} 

func (m *LocalEntityProto) Reset()   { *m = LocalEntityProto{} } 
func (m *LocalEntityProto) String() string { return proto.CompactTextString(m) } 
func (*LocalEntityProto) ProtoMessage() {} 

func (m *LocalEntityProto) GetKind() pb.EntityProto_Kind { 
    if m != nil && m.Kind != nil { 
     return *m.Kind 
    } 
    return pb.EntityProto_GD_CONTACT 
} 

func (m *LocalEntityProto) GetKindUri() string { 
    if m != nil && m.KindUri != nil { 
     return *m.KindUri 
    } 
    return "" 
} 

func (m *LocalEntityProto) GetProperty() []*pb.Property { 
    if m != nil { 
     return m.Property 
    } 
    return nil 
} 

func (m *LocalEntityProto) GetRawProperty() []*pb.Property { 
    if m != nil { 
     return m.RawProperty 
    } 
    return nil 
} 

func (m *LocalEntityProto) GetRank() int32 { 
    if m != nil && m.Rank != nil { 
     return *m.Rank 
    } 
    return 0 
}