詳盡 | PyTorch動(dòng)態(tài)圖解析

重磅干貨，第一時(shí)間送達(dá)

本文轉(zhuǎn)自：深度學(xué)習(xí)這件小事

   

    
   
   

    

     

     

     

     

     

     

     

    
    
void THPAutograd_initFunctions(){  THPObjectPtr module(PyModule_New("torch._C._functions"));  ......  generated::initialize_autogenerated_functions();  auto c_module = THPObjectPtr(PyImport_ImportModule("torch._C"));}
   
   

    

   

   

    

     

    
    
static std::unordered_map<std::type_index, THPObjectPtr> cpp_function_types
   
   

    

   

   

    

     

     

     

     

    
    
>>> gemfield = torch.empty([2,2],requires_grad=True)>>> syszux = gemfield * gemfield>>> syszux.grad_fn<ThMulBackward object at 0x7f111621c350>
   
   

    

   

   

    

     

     

     

     

     

    
    
gemfield = torch.ones(2, 2, requires_grad=True)syszux = gemfield + 2civilnet = syszux * syszux * 3gemfieldout = civilnet.mean()gemfieldout.backward()
   
   

    

   

   

    

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

    
    
#Variable實(shí)例gemfield --> grad_fn_ (Function實(shí)例）= None         --> grad_accumulator_ (Function實(shí)例）= AccumulateGrad實(shí)例0x55ca7f304500         --> output_nr_ = 0
#Function實(shí)例， 0x55ca7f872e90AddBackward0實(shí)例 --> sequence_nr_ (uint64_t) = 0            --> next_edges_ (edge_list) --> std::vector<Edge> = [(AccumulateGrad實(shí)例, 0)，(0, 0)]            --> input_metadata_ --> [(type, shape, device)...] = [(CPUFloatType, [2, 2],cpu])]            --> alpha (Scalar) = 1            --> apply() --> 使用 AddBackward0 的apply
#Variable實(shí)例syszux --> grad_fn_ (Function實(shí)例）= AddBackward0實(shí)例0x55ca7f872e90       --> output_nr_ = 0
#Function實(shí)例， 0x55ca7ebba2a0MulBackward0 --> sequence_nr_ (uint64_t) = 1            --> next_edges_ (edge_list) = [(AddBackward0實(shí)例0x55ca7f872e90,0),(AddBackward0實(shí)例0x55ca7f872e90,0)]            --> input_metadata_ --> [(type, shape, device)...] = [(CPUFloatType, [2, 2],cpu])]            --> alpha (Scalar) = 1            --> apply() --> 使用 MulBackward0 的apply
# #Variable實(shí)例，syszux * syszux得到的tmptmp --> grad_fn_ (Function實(shí)例）= MulBackward0實(shí)例0x55ca7ebba2a0    --> output_nr_ = 0
#Function實(shí)例，0x55ca7fada2f0MulBackward0 --> sequence_nr_ (uint64_t) = 2 （每個(gè)線(xiàn)程內(nèi)自增）            --> next_edges_ (edge_list) = [(MulBackward0實(shí)例0x55ca7ebba2a0,0),(0,0)]            --> input_metadata_ --> [(type, shape, device)...] = [(CPUFloatType, [2, 2],cpu])]            --> self_ (SavedVariable) = tmp的淺拷貝            --> other_ (SavedVariable) = 3的淺拷貝            --> apply() --> 使用 MulBackward0 的apply
#Variable實(shí)例civilnet --> grad_fn_ (Function實(shí)例）= MulBackward0實(shí)例0x55ca7fada2f0                                          -
#Function實(shí)例，0x55ca7eb358b0MeanBackward0 --> sequence_nr_ (uint64_t) = 3 （每個(gè)線(xiàn)程內(nèi)自增）              --> next_edges_ (edge_list) = [(MulBackward0實(shí)例0x55ca7fada2f0,0)]              --> input_metadata_ --> [(type, shape, device)...] = [(CPUFloatType|[]|cpu])]              --> self_sizes (std::vector<int64_t>) = (2, 2)              --> self_numel = 4              --> apply() --> 使用 MulBackward0 的apply#Variable實(shí)例gemfieldout --> grad_fn_ (Function實(shí)例）= MeanBackward0實(shí)例0x55ca7eb358b0            --> output_nr_ = 0
   

   

    

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

    
    
using edge_list = std::vector<Edge>;using variable_list = std::vector<Variable>;
struct TORCH_API Function {...  virtual variable_list apply(variable_list&& inputs) = 0;...  const uint64_t sequence_nr_;  edge_list next_edges_;  PyObject* pyobj_ = nullptr; // weak reference  std::unique_ptr<AnomalyMetadata> anomaly_metadata_ = nullptr;  std::vector<std::unique_ptr<FunctionPreHook>> pre_hooks_;  std::vector<std::unique_ptr<FunctionPostHook>> post_hooks_;  at::SmallVector<InputMetadata, 2> input_metadata_;};
   

   

    variable_list operator()(variable_list&& inputs) {
    
  return apply(std::move(inputs));
    
}
    

   

   

    

     

     

     

     

     

     

    
    
struct InputMetadata {...  const at::Type* type_ = nullptr;  at::DimVector shape_;  at::Device device_ = at::kCPU;};
   
   

    

   

   

    

     

     

     

     

     

    
    
struct Edge {...  std::shared_ptr<Function> function;  uint32_t input_nr;};
   
   

    

   

   

    

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

    
    
CopySlices : public Function DelayedError : public Function Error : public Function Gather : public Function GraphRoot : public Function Scatter : public FunctionAccumulateGrad : public Function AliasBackward : public Function AsStridedBackward : public Function CopyBackwards : public Function DiagonalBackward : public Function ExpandBackward : public Function IndicesBackward0 : public Function IndicesBackward1 : public Function PermuteBackward : public Function SelectBackward : public Function SliceBackward : public Function SqueezeBackward0 : public Function SqueezeBackward1 : public Function TBackward : public Function TransposeBackward0 : public Function UnbindBackward : public Function UnfoldBackward : public Function UnsqueezeBackward0 : public Function ValuesBackward0 : public Function ValuesBackward1 : public Function ViewBackward : public Function
PyFunction : public Function
   
   

    

   

   

    
   
   

    

     

     

     

     

     

    
    
struct AccumulateGrad : public Function {  explicit AccumulateGrad(Variable variable_);  variable_list apply(variable_list&& grads) override;  Variable variable;};
   
   

    

   

   

    
   
   

    

     

     

     

     

     

     

     

     

     

    
    
struct GraphRoot : public Function {  GraphRoot(edge_list functions, variable_list inputs)      : Function(std::move(functions)),        outputs(std::move(inputs)) {}  variable_list apply(variable_list&& inputs) override {    return outputs;  }  variable_list outputs;};
   
   

    

   

   

    struct TraceableFunction : public Function {
    
  using Function::Function;
    
  bool is_traceable() final {
    
    return true;
    
  }
    
};
    

   

   

    

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

    
    
AbsBackward : public TraceableFunction AcosBackward : public TraceableFunction AdaptiveAvgPool2DBackwardBackward : public TraceableFunction AdaptiveAvgPool2DBackward : public TraceableFunction AdaptiveAvgPool3DBackwardBackward : public TraceableFunction AdaptiveAvgPool3DBackward : public TraceableFunction AdaptiveMaxPool2DBackwardBackward : public TraceableFunction AdaptiveMaxPool2DBackward : public TraceableFunction AdaptiveMaxPool3DBackwardBackward : public TraceableFunction AdaptiveMaxPool3DBackward : public TraceableFunction AddBackward0 : public TraceableFunction AddBackward1 : public TraceableFunction AddbmmBackward : public TraceableFunction AddcdivBackward : public TraceableFunction AddcmulBackward : public TraceableFunction AddmmBackward : public TraceableFunction AddmvBackward : public TraceableFunction AddrBackward : public TraceableFunction ......SoftmaxBackwardDataBackward : public TraceableFunction SoftmaxBackward : public TraceableFunction ......UpsampleBicubic2DBackwardBackward : public TraceableFunction UpsampleBicubic2DBackward : public TraceableFunction UpsampleBilinear2DBackwardBackward : public TraceableFunction UpsampleBilinear2DBackward : public TraceableFunction UpsampleLinear1DBackwardBackward : public TraceableFunction UpsampleLinear1DBackward : public TraceableFunction UpsampleNearest1DBackwardBackward : public TraceableFunction UpsampleNearest1DBackward : public TraceableFunction UpsampleNearest2DBackwardBackward : public TraceableFunction UpsampleNearest2DBackward : public TraceableFunction UpsampleNearest3DBackwardBackward : public TraceableFunction UpsampleNearest3DBackward : public TraceableFunction UpsampleTrilinear3DBackwardBackward : public TraceableFunction UpsampleTrilinear3DBackward : public TraceableFunction ......
   
   

    

   

   

    

     

     

     

     

     

    
    
struct AddBackward0 : public TraceableFunction {  using TraceableFunction::TraceableFunction;  variable_list apply(variable_list&& grads) override;  Scalar alpha;};
   
   

    

   

   

    

     

     

     

     

     

    
    
gemfield = torch.ones(2, 2, requires_grad=True)syszux = gemfield + 2civilnet = syszux * syszux * 3gemfieldout = civilnet.mean()gemfieldout.backward()
   
   

    

   

   

    

     

     

     

     

     

     

     

     

     

     

     

     

     

    
    
struct Engine {  using ready_queue_type = std::deque<std::pair<std::shared_ptr<Function>, InputBuffer>>;  using dependencies_type = std::unordered_map<Function*, int>;  virtual variable_list execute(const edge_list& roots,const variable_list& inputs,...const edge_list& outputs = {});  void queue_callback(std::function<void()> callback);protected:  void compute_dependencies(Function* root, GraphTask& task);  void evaluate_function(FunctionTask& task);  void start_threads();  virtual void thread_init(int device);  virtual void thread_main(GraphTask *graph_task);  std::vector<std::shared_ptr<ReadyQueue>> ready_queues;};
   
   

    

   

   

    struct PythonEngine : public Engine
    

   

   

    

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

     

    
    
#torch/tensor.py，self is gemfieldoutdef backward(self, gradient=None, retain_graph=None, create_graph=False)|V#torch.autograd.backward(self, gradient, retain_graph, create_graph)#torch/autograd/__init__.pydef backward(tensors, grad_tensors=None, retain_graph=None, create_graph=False, grad_variables=None)|VVariable._execution_engine.run_backward(tensors, grad_tensors, retain_graph, create_graph,allow_unreachable=True)#轉(zhuǎn)化為Variable._execution_engine.run_backward((gemfieldout,), (tensor(1.),), False, False,True)|V#torch/csrc/autograd/python_engine.cppPyObject *THPEngine_run_backward(THPEngine *self, PyObject *args, PyObject *kwargs)|V#torch/csrc/autograd/python_engine.cppvariable_list PythonEngine::execute(const edge_list& roots, const variable_list& inputs, bool keep_graph, bool create_graph, const edge_list& outputs)|V#torch/csrc/autograd/engine.cpp
   

總結(jié)