1、历史汇总查询接口优化，非当日数据存入数据库，每1小时更新一次；
This commit is contained in:
parent
2eb326eb0a
commit
9cb7995515
|
|
@ -3150,3 +3150,107 @@ func (e MiGuDeployService) queryHistoricalDataByHour(startTime, endTime string,
|
|||
|
||||
return paginatedData, summaryData, len(filteredData), nil
|
||||
}
|
||||
|
||||
func (e MiGuDeployService) HistoricalSummaryListNewByCatch(c *gin.Context) {
|
||||
fmt.Println("HistoricalSummaryList")
|
||||
err := e.MakeContext(c).MakeOrm().Errors
|
||||
if err != nil {
|
||||
fmt.Println("MakeContext err:", err)
|
||||
e.Logger.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
// 请求参数绑定
|
||||
req := &models.HistoricalSummaryListReq{}
|
||||
if c.ShouldBindJSON(req) != nil {
|
||||
logger.Errorf("para err")
|
||||
response.Error(c, http.StatusBadRequest, errors.New("para err"), "参数错误")
|
||||
return
|
||||
}
|
||||
|
||||
// 分页处理
|
||||
if req.PageNum < 1 {
|
||||
req.PageNum = 1
|
||||
}
|
||||
if req.PageSize == 0 {
|
||||
req.PageSize = 10
|
||||
}
|
||||
|
||||
var summaries []models.MgHistoricalSummary
|
||||
|
||||
today := time.Now().Format("2006-01-02")
|
||||
|
||||
// 1. 先查缓存表（非今日）
|
||||
var cacheQuery *gorm.DB
|
||||
|
||||
// 如果 req.StartTime 和 req.EndTime 不为空，应用时间条件
|
||||
if req.StartTime != "" && req.EndTime != "" {
|
||||
startDate := req.StartTime[:10] // 取前 10 位 YYYY-MM-DD
|
||||
endDate := req.EndTime[:10] // 同上
|
||||
cacheQuery = e.Orm.Table("mg_historical_summary").
|
||||
Where("date >= ? AND date <= ?", startDate, endDate)
|
||||
} else {
|
||||
// 如果为空，则查询所有数据
|
||||
cacheQuery = e.Orm.Table("mg_historical_summary")
|
||||
}
|
||||
|
||||
if req.SkuCode != 0 {
|
||||
cacheQuery = cacheQuery.Where("product_id = ?", req.SkuCode)
|
||||
}
|
||||
|
||||
if req.Channel != "" {
|
||||
cacheQuery = cacheQuery.Where("channel_code = ?", req.Channel)
|
||||
}
|
||||
|
||||
var cacheData []models.MgHistoricalSummary
|
||||
if err := cacheQuery.Find(&cacheData).Error; err != nil {
|
||||
response.Error(c, http.StatusInternalServerError, err, "查询失败")
|
||||
}
|
||||
|
||||
summaries = append(summaries, cacheData...)
|
||||
|
||||
// 2. 如果包含今日，查实时数据
|
||||
if req.EndTime >= today || req.EndTime == "" {
|
||||
// 计算今天的开始时间和结束时间
|
||||
now := time.Now()
|
||||
// 格式化时间为 string 类型，格式为 "YYYY-MM-DD"
|
||||
start := now.Truncate(24 * time.Hour).Format("2006-01-02") // 今天的零点，字符串格式
|
||||
end := now.Truncate(24 * time.Hour).Add(24*time.Hour - time.Second).Format("2006-01-02") // 今天的23:59:59，字符串格式
|
||||
|
||||
realTimeData, err := models.CalculateDailySummaryFromRealtime(e.Orm, start, end, req.Channel, req.SkuCode)
|
||||
if err != nil {
|
||||
response.Error(c, http.StatusInternalServerError, err, "查询失败")
|
||||
}
|
||||
|
||||
for _, item := range realTimeData {
|
||||
summaries = append(summaries, item)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 排序：默认按日期倒序
|
||||
sort.SliceStable(summaries, func(i, j int) bool {
|
||||
return summaries[i].Date > summaries[j].Date
|
||||
})
|
||||
|
||||
// 4. 分页
|
||||
total := len(summaries)
|
||||
start := (req.PageNum - 1) * req.PageSize
|
||||
end := start + req.PageSize
|
||||
if start > total {
|
||||
start = total
|
||||
}
|
||||
if end > total {
|
||||
end = total
|
||||
}
|
||||
|
||||
pageList := summaries[start:end]
|
||||
|
||||
resp := &models.HistoricalSummaryListResp{
|
||||
List: pageList,
|
||||
Count: total,
|
||||
PageNum: req.PageNum,
|
||||
PageSize: req.PageSize,
|
||||
}
|
||||
|
||||
e.OK(resp, "")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -139,6 +139,8 @@ type MgHourSummary struct {
|
|||
|
||||
// MgHistoricalSummary 历史汇总查询表对应的结构体
|
||||
type MgHistoricalSummary struct {
|
||||
Model
|
||||
|
||||
Date string `json:"date"` // 日期
|
||||
ProductID int64 `json:"product_id"` // 产品ID
|
||||
ChannelCode string `gorm:"size:255" json:"channel_code"` // 渠道编码
|
||||
|
|
@ -2140,3 +2142,126 @@ func GetMonthlyEffectiveUserStats(db *gorm.DB, retentionMonth string, skuCode in
|
|||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
func UpdateHistoricalSummaryCache() {
|
||||
logger.Info("****** UpdateHistoricalSummaryCache start ******")
|
||||
fmt.Println("****** UpdateHistoricalSummaryCache start ******")
|
||||
if database.Db == nil {
|
||||
logger.Error("Database connection is nil")
|
||||
fmt.Println("Database connection is nil")
|
||||
return
|
||||
}
|
||||
|
||||
startTime := "1970-01-01 00:00:00"
|
||||
endTime := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + " 23:59:59"
|
||||
|
||||
// 1. 查询某天的完整实时数据
|
||||
summaries, err := CalculateDailySummaryFromRealtime(database.Db, startTime, endTime, "", 0)
|
||||
if err != nil {
|
||||
log.Printf("calculateDailySummaryFromRealtime failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(summaries) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
tx := database.Db.Begin()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
// 2. 清空表中的所有数据
|
||||
if err = tx.Exec("TRUNCATE TABLE mg_historical_summary").Error; err != nil {
|
||||
logger.Error("TRUNCATE TABLE mg_historical_summary error:", err)
|
||||
fmt.Println("TRUNCATE TABLE mg_historical_summary error")
|
||||
tx.Rollback()
|
||||
return
|
||||
}
|
||||
|
||||
// 3. 插入新数据到缓存表
|
||||
if err = tx.Create(&summaries).Error; err != nil {
|
||||
logger.Error("UpdateHistoricalSummaryCache Create error:", err)
|
||||
fmt.Println("UpdateHistoricalSummaryCache Create error")
|
||||
tx.Rollback()
|
||||
return
|
||||
}
|
||||
|
||||
err = tx.Commit().Error
|
||||
if err != nil {
|
||||
logger.Error("UpdateHistoricalSummaryCache Commit error:", err)
|
||||
fmt.Println("UpdateHistoricalSummaryCache Commit error")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func CalculateDailySummaryFromRealtime(db *gorm.DB, startTime, endTime, channelCode string, productID int) ([]MgHistoricalSummary, error) {
|
||||
|
||||
var results []MgHistoricalSummary
|
||||
|
||||
qs := db.Model(&MgOrder{}).
|
||||
Select(`
|
||||
DATE_FORMAT(mg_order.subscribe_time, '%Y-%m-%d') AS date,
|
||||
mg_order.product_id,
|
||||
mg_order.channel_code,
|
||||
IFNULL(submission_count.submission_count, 0) AS submission_count,
|
||||
COUNT(CASE WHEN mg_order.is_one_hour_cancel = 1 THEN 1 END) AS new_user_unsub_within_hour,
|
||||
COUNT(CASE WHEN mg_order.state = 2 AND DATE(mg_order.unsubscribe_time) = DATE(mg_order.subscribe_time) THEN 1 END) AS new_user_unsub_on_day,
|
||||
COUNT(CASE WHEN mg_order.state = 2 AND TIMESTAMPDIFF(HOUR, mg_order.subscribe_time, mg_order.unsubscribe_time) <= 24 THEN 1 END) AS new_user_unsub_within_24h,
|
||||
COUNT(*) AS new_user_count,
|
||||
SUM(CASE WHEN mg_order.state = 2 THEN 1 ELSE 0 END) AS total_new_user_unsub,
|
||||
CONCAT(ROUND(COUNT(CASE WHEN mg_order.is_one_hour_cancel = 1 THEN 1 END) * 100.0 / NULLIF(COUNT(*), 0), 2), '%') AS new_user_unsub_within_hour_rate,
|
||||
CONCAT(ROUND(COUNT(CASE WHEN mg_order.state = 2 AND DATE(mg_order.unsubscribe_time) = DATE(mg_order.subscribe_time) THEN 1 END) * 100.0 / NULLIF(COUNT(*), 0), 2), '%') AS new_user_unsub_on_day_rate,
|
||||
CONCAT(ROUND(COUNT(CASE WHEN mg_order.state = 2 AND TIMESTAMPDIFF(HOUR, mg_order.subscribe_time, mg_order.unsubscribe_time) <= 24 THEN 1 END) * 100.0 / NULLIF(COUNT(*), 0), 2), '%') AS new_user_unsub_within_24h_rate,
|
||||
CONCAT(ROUND(SUM(CASE WHEN mg_order.state = 2 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0), 2), '%') AS total_new_user_unsub_rate,
|
||||
IFNULL(
|
||||
CONCAT(
|
||||
LEAST(
|
||||
ROUND(COUNT(*) * 100.0 / NULLIF(submission_count.submission_count, 0), 2),
|
||||
100.00
|
||||
),
|
||||
'%'
|
||||
),
|
||||
'0.00%'
|
||||
) AS submission_success_rate
|
||||
`).
|
||||
// 使用 Joins 来替代 LeftJoin 进行左连接
|
||||
Joins(`
|
||||
LEFT JOIN (
|
||||
SELECT
|
||||
channel_code,
|
||||
DATE(created_at) AS created_date,
|
||||
COUNT(*) AS submission_count
|
||||
FROM mg_transaction_log
|
||||
WHERE verification_code != ''
|
||||
GROUP BY channel_code, created_date
|
||||
) AS submission_count
|
||||
ON submission_count.channel_code = mg_order.channel_code
|
||||
AND submission_count.created_date = DATE(mg_order.subscribe_time)
|
||||
`).
|
||||
Where("mg_order.subscribe_time BETWEEN ? AND ?", startTime, endTime).
|
||||
Group("DATE(mg_order.subscribe_time), mg_order.product_id, mg_order.channel_code").
|
||||
Order("mg_order.subscribe_time DESC")
|
||||
|
||||
// 🔍 条件筛选：渠道
|
||||
if channelCode != "" {
|
||||
qs = qs.Where("mg_order.channel_code = ?", channelCode)
|
||||
}
|
||||
|
||||
// 🔍 条件筛选：产品
|
||||
if productID != 0 {
|
||||
qs = qs.Where("mg_order.product_id = ?", productID)
|
||||
}
|
||||
|
||||
err := qs.Find(&results).Error
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,9 +32,11 @@ func registerMiGuControlManageRouter(v1 *gin.RouterGroup, authMiddleware *jwt.Gi
|
|||
api.POST("home/data", apiMiGu.HomepageDataSummary) // 查询首页汇总数据
|
||||
api.POST("home/revenue_analysis", apiMiGu.CalculateRevenueAnalysis) // 营收分析
|
||||
|
||||
api.POST("historical_summary/list", apiMiGu.HistoricalSummaryListOld) // 历史汇总查询
|
||||
//api.POST("historical_summary/list", apiMiGu.HistoricalSummaryListOld) // 历史汇总查询
|
||||
//api.POST("order/import", apiMiGu.ImportExcelToMgOrderHandler) // 通过excel导入订单数据
|
||||
//api.POST("order/import_update", apiMiGu.ImportExcelToMgOrderHandlerUpdate) // 通过excel导入订单退订数据
|
||||
|
||||
api.POST("historical_summary/list", apiMiGu.HistoricalSummaryListNewByCatch) // 历史汇总查询（缓存版本）
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -100,8 +100,14 @@ func run() error {
|
|||
fmt.Println("err:", err)
|
||||
}
|
||||
} else {
|
||||
// 每小时更新1次历史汇总数据
|
||||
err := s.Every(1).Hour().Do(models.UpdateHistoricalSummaryCache)
|
||||
if err != nil {
|
||||
fmt.Println("err:", err)
|
||||
}
|
||||
|
||||
// 统一每天定时任务
|
||||
err := s.Every(1).Day().At("01:30").Do(func() {
|
||||
err = s.Every(1).Day().At("01:30").Do(func() {
|
||||
if time.Now().Day() == 1 {
|
||||
// 每月1号的逻辑
|
||||
fmt.Println("****每月1号 检测未退订 任务延迟到03:30执行****")
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ settings:
|
|||
# 数据库类型 mysql, sqlite3, postgres, sqlserver
|
||||
driver: mysql
|
||||
# 数据库连接字符串 mysql 缺省信息 charset=utf8&parseTime=True&loc=Local&timeout=1000ms
|
||||
source:
|
||||
source: migu_pro:RhDShkE4WabxXazb@tcp(112.33.14.191:3306)/migu_pro?charset=utf8&parseTime=True&loc=Local&timeout=1000ms
|
||||
gen:
|
||||
# 代码生成读取的数据库名称
|
||||
dbname: dbname
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user