Merge pull request #583 from dadosjusbr/manual-collection-data

feat: add support for manual data collection
dadosjusbr · Oct 28, 2024 · e969098 · e969098
2 parents 7c9861c + ecc7f43
commit e969098
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 29 deletions.
diff --git a/docs/docs.go b/docs/docs.go
@@ -1173,6 +1173,9 @@ const docTemplate = `{
                 "ano": {
                     "type": "integer"
                 },
+                "coleta_manual": {
+                    "type": "boolean"
+                },
                 "dados_coleta": {
                     "$ref": "#/definitions/papi.collect"
                 },

diff --git a/docs/swagger.json b/docs/swagger.json
@@ -1164,6 +1164,9 @@
                 "ano": {
                     "type": "integer"
                 },
+                "coleta_manual": {
+                    "type": "boolean"
+                },
                 "dados_coleta": {
                     "$ref": "#/definitions/papi.collect"
                 },

diff --git a/docs/swagger.yaml b/docs/swagger.yaml
@@ -270,6 +270,8 @@ definitions:
     properties:
       ano:
         type: integer
+      coleta_manual:
+        type: boolean
       dados_coleta:
         $ref: '#/definitions/papi.collect'
       error:

diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.18
 
 require (
 	github.com/dadosjusbr/proto v0.0.0-20221212025627-91c60aa3cd12
-	github.com/dadosjusbr/storage v0.0.0-20240913213102-72765cc03b4e
+	github.com/dadosjusbr/storage v0.0.0-20240923172949-d3b675a3e292
 	github.com/gocarina/gocsv v0.0.0-20220712153207-8b2118da4570
 	github.com/golang/mock v1.6.0
 	github.com/joho/godotenv v1.4.0

diff --git a/go.sum b/go.sum
@@ -41,6 +41,8 @@ github.com/dadosjusbr/storage v0.0.0-20240514131514-43ac5da3ae8a h1:1LyzfsNzKgLj
 github.com/dadosjusbr/storage v0.0.0-20240514131514-43ac5da3ae8a/go.mod h1:rIM/dbZMdrMfVnZgNgRNRRtsxfhSMH8S8X7MZEeKkrQ=
 github.com/dadosjusbr/storage v0.0.0-20240913213102-72765cc03b4e h1:RNcbmof3iPyJQNWbEPkNyKKffmQ8jkZa8QH/vZ5eJP0=
 github.com/dadosjusbr/storage v0.0.0-20240913213102-72765cc03b4e/go.mod h1:rIM/dbZMdrMfVnZgNgRNRRtsxfhSMH8S8X7MZEeKkrQ=
+github.com/dadosjusbr/storage v0.0.0-20240923172949-d3b675a3e292 h1:cgrdaqWjNN9enNcWPaPtqWFbxo1bLWz9C/Lxhke4WM4=
+github.com/dadosjusbr/storage v0.0.0-20240923172949-d3b675a3e292/go.mod h1:rIM/dbZMdrMfVnZgNgRNRRtsxfhSMH8S8X7MZEeKkrQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

diff --git a/papi/handlers.go b/papi/handlers.go
@@ -237,7 +237,9 @@ func (h handler) GetMonthlyInfo(c echo.Context) error {
 							CrawlerVersion: mi.CrawlerVersion,
 							ParserRepo:     mi.ParserRepo,
 							ParserVersion:  mi.ParserVersion,
-						}})
+						},
+						ManualCollection: mi.ManualCollection,
+					})
 				// The status 4 is a report from crawlers that data is unavailable or malformed. By removing them from the API results, we make sure they are displayed as if there is no data.
 			} else if mi.ProcInfo.Status != 4 {
 				sumMI = append(
@@ -249,11 +251,13 @@ func (h handler) GetMonthlyInfo(c echo.Context) error {
 							Status:       mi.ProcInfo.Status,
 							Cmd:          mi.ProcInfo.Cmd,
 						},
-						Month:    mi.Month,
-						Year:     mi.Year,
-						Package:  nil,
-						Summary:  nil,
-						Metadata: nil})
+						Month:            mi.Month,
+						Year:             mi.Year,
+						Package:          nil,
+						Summary:          nil,
+						Metadata:         nil,
+						ManualCollection: mi.ManualCollection,
+					})
 			}
 		}
 	}
@@ -369,6 +373,7 @@ func (h handler) V2GetMonthlyInfo(c echo.Context) error {
 					ParserRepo:     monthlyInfo.ParserRepo,
 					ParserVersion:  monthlyInfo.ParserVersion,
 				},
+				ManualCollection: monthlyInfo.ManualCollection,
 			}
 		//O status 4 informa que os dados estão indisponíveis. Ao removê-los dos resultados da API, garantimos que eles sejam exibidos como se não houvesse dados.
 	} else if monthlyInfo.ProcInfo.Status != 4 {
@@ -379,11 +384,12 @@ func (h handler) V2GetMonthlyInfo(c echo.Context) error {
 				Status:       monthlyInfo.ProcInfo.Status,
 				Cmd:          monthlyInfo.ProcInfo.Cmd,
 			},
-			Month:    monthlyInfo.Month,
-			Year:     monthlyInfo.Year,
-			Package:  nil,
-			Summary:  nil,
-			Metadata: nil,
+			Month:            monthlyInfo.Month,
+			Year:             monthlyInfo.Year,
+			Package:          nil,
+			Summary:          nil,
+			Metadata:         nil,
+			ManualCollection: monthlyInfo.ManualCollection,
 		}
 	} else {
 		return c.NoContent(http.StatusNoContent)
@@ -498,7 +504,9 @@ func (h handler) GetMonthlyInfosByYear(c echo.Context) error {
 							CrawlerVersion: mi.CrawlerVersion,
 							ParserRepo:     mi.ParserRepo,
 							ParserVersion:  mi.ParserVersion,
-						}})
+						},
+						ManualCollection: mi.ManualCollection,
+					})
 				//O status 4 informa que os dados estão indisponíveis. Ao removê-los dos resultados da API, garantimos que eles sejam exibidos como se não houvesse dados.
 			} else if mi.ProcInfo.Status != 4 {
 				sumMI = append(
@@ -510,11 +518,13 @@ func (h handler) GetMonthlyInfosByYear(c echo.Context) error {
 							Status:       mi.ProcInfo.Status,
 							Cmd:          mi.ProcInfo.Cmd,
 						},
-						Month:    mi.Month,
-						Year:     mi.Year,
-						Package:  nil,
-						Summary:  nil,
-						Metadata: nil})
+						Month:            mi.Month,
+						Year:             mi.Year,
+						Package:          nil,
+						Summary:          nil,
+						Metadata:         nil,
+						ManualCollection: mi.ManualCollection,
+					})
 			}
 		}
 	}
@@ -684,7 +694,7 @@ func (h handler) V2GetAggregateIndexesWithParams(c echo.Context) error {
 //	@Produce		json
 //	@Success		200			{object}	[]aggregateIndexesByGroup	"Requisição bem sucedida."
 //	@Failure		500			{string}	string						"Erro interno do servidor."
-//	@Router			/v2/indice 																																																																																													[get]
+//	@Router			/v2/indice 																																																																																																					[get]
 func (h handler) V2GetAggregateIndexes(c echo.Context) error {
 	agregado := c.QueryParam("agregado")
 	detalhe := c.QueryParam("detalhe")
@@ -876,7 +886,9 @@ func (h handler) V2GetAllAgencyInformation(c echo.Context) error {
 					Score:             c.Score.Score,
 					CompletenessScore: c.Score.CompletenessScore,
 					EasinessScore:     c.Score.EasinessScore,
-				}})
+				},
+				ManualCollection: c.ManualCollection,
+			})
 			numMonthsWithData++
 		}
 		aggregateScore += c.Score.Score

diff --git a/papi/models.go b/papi/models.go
@@ -72,15 +72,16 @@ type miError struct {
 }
 
 type summaryzedMI struct {
-	AgencyID string     `json:"id_orgao,omitempty"`
-	Month    int        `json:"mes,omitempty"`
-	Year     int        `json:"ano,omitempty"`
-	Summary  *summaries `json:"sumarios,omitempty"`
-	Package  *backup    `json:"pacote_de_dados,omitempty"`
-	Metadata *metadata  `json:"metadados,omitempty"`
-	Score    *score     `json:"indice_transparencia,omitempty"`
-	Collect  *collect   `json:"dados_coleta,omitempty"`
-	Error    *miError   `json:"error,omitempty"`
+	AgencyID         string     `json:"id_orgao,omitempty"`
+	Month            int        `json:"mes,omitempty"`
+	Year             int        `json:"ano,omitempty"`
+	Summary          *summaries `json:"sumarios,omitempty"`
+	Package          *backup    `json:"pacote_de_dados,omitempty"`
+	Metadata         *metadata  `json:"metadados,omitempty"`
+	Score            *score     `json:"indice_transparencia,omitempty"`
+	Collect          *collect   `json:"dados_coleta,omitempty"`
+	ManualCollection bool       `json:"coleta_manual"`
+	Error            *miError   `json:"error,omitempty"`
 }
 
 type agency struct {